From e661df6bb45fe32b0066bc209fdf401aa319cd3f Mon Sep 17 00:00:00 2001 From: Scott Gordon Date: Wed, 16 Jul 2025 15:23:07 +0100 Subject: [PATCH 01/21] vdi module --- community/modules/scripts/vdi-setup/README.md | 164 ++++++++++++++ community/modules/scripts/vdi-setup/main.tf | 137 +++++++++++ .../modules/scripts/vdi-setup/outputs.tf | 59 +++++ .../roles/base_os/tasks/gpu_driver_setup.yaml | 74 ++++++ .../vdi-setup/roles/base_os/tasks/main.yaml | 104 +++++++++ .../roles/secret_manager/defaults/main.yaml | 16 ++ .../roles/secret_manager/tasks/main.yaml | 102 +++++++++ .../tasks/user_secret_tasks.yaml | 127 +++++++++++ .../user_provision/tasks/local_users.yaml | 51 +++++ .../roles/user_provision/tasks/main.yaml | 40 ++++ .../roles/user_provision/tasks/os_login.yaml | 15 ++ .../roles/vdi_tool/tasks/guacamole.yaml | 212 ++++++++++++++++++ .../vdi-setup/roles/vdi_tool/tasks/main.yaml | 26 +++ .../roles/vdi_tool/tasks/nomachine.yaml | 15 ++ .../roles/vdi_tool/tasks/workspot.yaml | 15 ++ .../templates/connection_bootstrap.sql.j2 | 72 ++++++ .../vdi_tool/templates/user_bootstrap.sql.j2 | 22 ++ .../vdi-setup/roles/vnc/tasks/main.yaml | 23 ++ .../vdi-setup/roles/vnc/tasks/tigervnc.yaml | 69 ++++++ .../vdi-setup/roles/vnc/tasks/tightvnc.yaml | 15 ++ .../roles/vnc/tasks/vnc_user_config.yaml | 59 +++++ .../vdi-setup/roles/vnc/templates/config.j2 | 2 + .../roles/vnc/templates/vncserver.users.j2 | 3 + .../vnc/templates/vncserver@.service.rocky.j2 | 15 ++ .../templates/vncserver@.service.ubuntu.j2 | 16 ++ .../vdi-setup/templates/install.yaml.tftpl | 12 + .../vdi-setup/templates/vars.yaml.tftpl | 23 ++ .../modules/scripts/vdi-setup/validation.tf | 53 +++++ .../modules/scripts/vdi-setup/variables.tf | 103 +++++++++ 29 files changed, 1644 insertions(+) create mode 100644 community/modules/scripts/vdi-setup/README.md create mode 100644 community/modules/scripts/vdi-setup/main.tf create mode 100644 community/modules/scripts/vdi-setup/outputs.tf create mode 100644 community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/secret_manager/defaults/main.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/user_provision/tasks/os_login.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/nomachine.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/workspot.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/vdi_tool/templates/connection_bootstrap.sql.j2 create mode 100644 community/modules/scripts/vdi-setup/roles/vdi_tool/templates/user_bootstrap.sql.j2 create mode 100644 community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/vnc/tasks/tigervnc.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/vnc/tasks/tightvnc.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/vnc/tasks/vnc_user_config.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/vnc/templates/config.j2 create mode 100644 community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver.users.j2 create mode 100644 community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.rocky.j2 create mode 100644 community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.ubuntu.j2 create mode 100644 community/modules/scripts/vdi-setup/templates/install.yaml.tftpl create mode 100644 community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl create mode 100644 community/modules/scripts/vdi-setup/validation.tf create mode 100644 community/modules/scripts/vdi-setup/variables.tf diff --git a/community/modules/scripts/vdi-setup/README.md b/community/modules/scripts/vdi-setup/README.md new file mode 100644 index 0000000000..b4dc34a00f --- /dev/null +++ b/community/modules/scripts/vdi-setup/README.md @@ -0,0 +1,164 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Description + +Mostly tested with Rocky. Working on Debian and Ubuntu images (commented in blueprint example below). +Distro / VDI 'flavour' variations are handled in the 'base_os' role for the most part. + +### Service Startup and Health Checking + +The VDI module includes basic health checks to ensure services start properly: + +### Secret Manager Integration + +The VDI module supports flexible Secret Manager integration: + +- **Default Behavior**: Secrets are stored in the deployment project +- **Cross-Project Secrets**: Use `secret_project` to specify a different GCP project for user secrets +- **Automatic Password Generation**: If no `secret_name` is provided, random passwords are generated and stored +- **Existing Secret Retrieval**: Provide `secret_name` to use existing secrets from Secret Manager + +- **Database Health Check**: Waits for PostgreSQL to be ready on port 5432 (60 second timeout) +- **Guacamole Web Interface**: Checks HTTP endpoint availability +- **API Verification**: Verifies Guacamole admin login functionality +- **VNC Service Restart**: Systemd services configured with automatic restart on failure + +## Outputs + +The module provides the following outputs: + +- `guacamole_url`: The URL to access the Guacamole web interface (null until VM is created) +- `guacamole_admin_username`: The admin username for Guacamole (default: "guacadmin") +- `guacamole_admin_password_secret`: The name of the Secret Manager secret containing the Guacamole admin password + - `vdi_user_credentials`: A map of VDI user credentials stored in Secret Manager, including: + - `username`: The VDI user's username + - `port`: The VNC port assigned to the user + - `secret_name`: The name of the Secret Manager secret containing the user's password + - `secret_project`: The GCP project where the user's secret is stored (defaults to deployment project) +- `vdi_instance_ip`: The IP address of the VDI instance (null until VM is created) +- `vdi_instance_name`: The name of the VDI instance (null until VM is created) + +## Basic Example (Guacamole) + +``` +blueprint_name: vdi-test + +vars: + deployment_name: vdi-test + project_id: * project name here * + region: us-central1 + zone: us-central1-a + +deployment_groups: +- group: primary + modules: + - id: network1 + source: modules/network/vpc + settings: + extra_iap_ports: [8080] + firewall_rules: + - name: allow-guacamole-8080-ext + description: Allow external ingress to Guacamole on TCP port 8080 + direction: INGRESS + ranges: ["0.0.0.0/0"] # or rev proxy address? + allow: + - protocol: tcp + ports: ["8080"] + + - id: enable-apis + source: community/modules/project/service-enablement + settings: + gcp_service_list: + - secretmanager.googleapis.com + - storage.googleapis.com + - compute.googleapis.com + + - id: vdi-setup + source: community/modules/scripts/vdi-setup + settings: + vnc_flavor: tigervnc + vdi_tool: guacamole + user_provision: local_users + vdi_user_group: vdiusers + vdi_resolution: 1920x1080 + vdi_users: + # Alice: password generated and saved to Secret Manager in deployment project + - username: alice + port: 5901 + # Bob: existing password is retrieved from Secret Manager in deployment project + - username: bob + port: 5902 + secret_name: a-password-for-bob + # Charlie: password retrieved from Secret Manager in a different project + - username: charlie + port: 5903 + secret_name: charlie-password + secret_project: another-project-id + + - id: guac_vm + source: modules/compute/vm-instance + settings: + instance_image: + # Several spported image families: + family: hpc-rocky-linux-8 + project: cloud-hpc-image-public + #family: debian-11 + #project: debian-cloud + #family: ubuntu-2204-lts + #project: ubuntu-os-cloud + machine_type: e2-highcpu-8 + tags: ["guacamole"] + use: + - network1 + - vdi-setup +``` + +[NVIDIA vWS drivers](https://cloud.google.com/compute/docs/gpus/grid-drivers-table) will be automatically installed on [supported machine types](https://cloud.google.com/compute/docs/gpus#gpu-virtual-workstations). + +Important note: Before deploying the above example you would need to ensure the `secret_name` exists in your local project, and that your service account has sufficient access permissions if retrieving the secret from a separate `secret_project`. The following two example commands show how you can create the two example user's respective secrets: + +```bash +# Create secrets for each user in the deployment project +echo -n "BobPassword123" | gcloud secrets create a-password-for-bob --data-file=- + +# Or create secrets in a different project (if using secret_project) +echo -n "CharliePassword123" | gcloud secrets create charlie-password --data-file=- --project=another-project-id +``` + +### Accessing Guacamole VDI + +After deployment, you can access the VDI in several ways: + +1. **Guacamole Web Interface**: + - Access web interface: + - http://$VM_PUBLIC_IP:8080/guacamole/#/ + - Note: It is not advisable to serve Guacamole directly from a public IP in production environments. Consider serving your VDI from behind a reverse proxy, load balancer, or tunnel to it directly over IAP (see below). + + - Admin credentials: + - Username: `guacadmin` + - Password: Retrieve the `webapp-server...` secret from Secret Manager + +2. **User VDI Access**: + - Each user's credentials are stored in Secret Manager + - Secret names follow the pattern: `vdi-user-password-{username}-{deployment_name}` for auto-generated passwords + - Or use the custom secret name specified in the `vdi_users` configuration + +3. **IAP Tunnel** (for development/testing): + ``` + gcloud compute start-iap-tunnel vdi-test-0 8080 \ + --local-host-port=localhost:8080 \ + --zone=us-central1-a + ``` + - Guacamole will then be accessible from http://localhost:8080/guacamole/ diff --git a/community/modules/scripts/vdi-setup/main.tf b/community/modules/scripts/vdi-setup/main.tf new file mode 100644 index 0000000000..a432fea15a --- /dev/null +++ b/community/modules/scripts/vdi-setup/main.tf @@ -0,0 +1,137 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + labels = merge(var.labels, { ghpc_module = "vdi-setup", ghpc_role = "scripts" }) +} + +# Create a tar.gz of roles/ directory +data "archive_file" "roles_tar" { + type = "tar.gz" + source_dir = "${path.module}/roles" + output_path = "${path.module}/roles.tar.gz" +} + +# Generate vars content using templatefile +locals { + vdi_vars_content = templatefile("${path.module}/templates/vars.yaml.tftpl", { + deployment_name = var.deployment_name + project_id = var.project_id + user_provision = var.user_provision + vnc_flavor = var.vnc_flavor + vdi_tool = var.vdi_tool + vdi_user_group = var.vdi_user_group + vdi_webapp_port = var.vdi_webapp_port + vdi_resolution = var.vdi_resolution + vdi_users = var.vdi_users + }) +} + +# Assemble runners +locals { + runners = [ + # Install dependencies + { + type = "shell" + destination = "install-deps.sh" + content = <<-EOT + #!/bin/bash + set -eux + /usr/local/ghpc-venv/bin/python3 -m pip install requests google-auth docker + ansible-galaxy collection install google.cloud + EOT + }, + # Stage roles.tar.gz + { + type = "data" + source = data.archive_file.roles_tar.output_path + destination = "/tmp/vdi/roles.tar.gz" + }, + + # Unpack into /tmp/vdi/roles + { + type = "shell" + destination = "unpack_roles.sh" + content = <<-EOT + #!/bin/bash + set -eux + mkdir -p /tmp/vdi/roles + tar xzf /tmp/vdi/roles.tar.gz -C /tmp/vdi/roles + EOT + }, + + # write out vars file as YAML + { + type = "data" + content = local.vdi_vars_content + destination = "/tmp/vdi/vars.yaml" + }, + + # Run the rendered playbook via ansible-local + { + type = "ansible-local" + content = templatefile("${path.module}/templates/install.yaml.tftpl", + { + roles = ["base_os", "secret_manager", "user_provision", "vnc", "vdi_tool"], + } + ) + destination = "/tmp/vdi/install.yaml" + # Todo: turn off debug and '-v' later: + args = "--extra-vars @/tmp/vdi/vars.yaml -v --extra-vars debug=true" + }, + # Todo: another runner here to delete /tmp/vdi afterwards? + ] + + required_apis = [ + "compute.googleapis.com", + "secretmanager.googleapis.com", + ] + + bucket_name = "${substr(var.deployment_name,0,39)}-vdi-scripts-${substr(md5(var.deployment_name),0,8)}" +} + +# Bucket to stage runners +resource "google_storage_bucket" "bucket" { + labels = local.labels + project = var.project_id + name = local.bucket_name + location = var.region + uniform_bucket_level_access = true + storage_class = "REGIONAL" +} + +# Use the startup-script module to push and execute them +module "startup_script" { + labels = local.labels + source = "../../../../modules/scripts/startup-script" + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + + runners = local.runners + gcs_bucket_path = "gs://${google_storage_bucket.bucket.name}" + + docker = { + enabled = true + } +} + +# Expose the combined startup script +locals { + combined_runner = { + type = "shell" + content = module.startup_script.startup_script + destination = "install-vdi-and-setup.sh" + } +} diff --git a/community/modules/scripts/vdi-setup/outputs.tf b/community/modules/scripts/vdi-setup/outputs.tf new file mode 100644 index 0000000000..a38dfbf8f4 --- /dev/null +++ b/community/modules/scripts/vdi-setup/outputs.tf @@ -0,0 +1,59 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "startup_script" { + description = "Combined startup script that installs VDI (VNC, Guacamole, users)." + value = module.startup_script.startup_script +} + +output "vdi_runner" { + description = "Shell runner wrapping Ansible playbook + roles (for custom-image or direct use)." + value = local.combined_runner +} + +output "guacamole_url" { + description = "The URL to access the Guacamole web interface" + value = var.vdi_instance_ip != null ? "http://${var.vdi_instance_ip}:${var.vdi_webapp_port}/guacamole/" : null +} + +output "guacamole_admin_username" { + description = "The admin username for Guacamole" + value = "guacadmin" +} + +output "guacamole_admin_password_secret" { + description = "The name of the Secret Manager secret containing the Guacamole admin password" + value = "webapp-server-password-${var.deployment_name}" +} + +output "vdi_user_credentials" { + description = "Map of VDI user credentials stored in Secret Manager" + value = { + for user in var.vdi_users : user.username => { + username = user.username + port = user.port + secret_name = user.secret_name != null ? user.secret_name : "vdi-user-password-${user.username}-${var.deployment_name}" + } + } +} + +output "vdi_instance_ip" { + description = "The IP address of the VDI instance" + value = var.vdi_instance_ip +} + +output "vdi_instance_name" { + description = "The name of the VDI instance" + value = var.vdi_instance_name +} diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml new file mode 100644 index 0000000000..eb2ff0ddb2 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml @@ -0,0 +1,74 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NVIDIA vWS (GRID) driver setup. See: +# https://cloud.google.com/compute/docs/gpus#gpu-virtual-workstations +# https://cloud.google.com/compute/docs/gpus/grid-drivers-table + +--- + +- name: Get GCP machine type from metadata server + ansible.builtin.uri: + url: http://metadata.google.internal/computeMetadata/v1/instance/machine-type + method: GET + headers: + Metadata-Flavor: Google + return_content: yes + register: gcp_machine_type + +- name: Extract short machine type + ansible.builtin.set_fact: + gcp_machine_type: "{{ gcp_machine_type.content.split('/')[-1] }}" + +- name: Extract machine family from machine type + ansible.builtin.set_fact: + machine_family: "{{ gcp_machine_type.split('-')[0] }}" + +- name: Check if machine type is supported for NVIDIA vWS + ansible.builtin.set_fact: + is_nvidia_vws_supported: "{{ machine_family in ['g2', 'n1'] }}" + +# Additional drivers for other machine type families can go here +- name: Set driver download URL based on machine type + ansible.builtin.set_fact: + driver_url: >- + {%- if is_nvidia_vws_supported -%} + https://storage.googleapis.com/nvidia-drivers-us-public/GRID/vGPU18.1/NVIDIA-Linux-x86_64-570.133.20-grid.run + {%- else -%} + + {%- endif %} + +# Block for NVIDIA vWS driver setup +- block: + - name: Install NVIDIA Vulkan ICD loader + ansible.builtin.package: + name: vulkan-loader + state: present + retries: 10 + delay: 3 + when: ansible_facts.os_family == 'RedHat' + - name: Download NVIDIA vWS driver + ansible.builtin.uri: + url: "{{ driver_url }}" + method: GET + dest: /tmp/nvidia-driver.run + mode: '0755' + status_code: + - 200 + - 304 + - name: Install NVIDIA vWS driver + ansible.builtin.command: + cmd: /tmp/nvidia-driver.run --dkms --silent + creates: /usr/bin/nvidia-smi + when: is_nvidia_vws_supported \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml new file mode 100644 index 0000000000..0fec11a329 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml @@ -0,0 +1,104 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Gather distribution facts + ansible.builtin.setup: + gather_subset: platform + +# GPU / vWS driver setup +- name: Check if NVIDIA GPU is present + ansible.builtin.shell: lspci | grep -i nvidia + register: nvidia_gpu_present + failed_when: false + +- name: Set package manager and desktop-packages + ansible.builtin.set_fact: + # pkg_mgr: "{{ 'yum' if ansible_facts.os_family == 'RedHat' else 'apt' }}" + gui_packages: >- + {{ + ['@Xfce','@base-x'] + if ansible_facts.os_family == 'RedHat' + else + ( + ['xfce4', 'xserver-xorg-core', 'dbus-x11'] + if ansible_facts.distribution == 'Debian' + else + ['xfce4', 'xserver-xorg-core'] + ) + }} + vnc_pkg: >- + {%- if vnc_flavor|lower == 'tigervnc' and ansible_facts.os_family == 'RedHat' -%} + tigervnc-server + {%- elif vnc_flavor|lower == 'tigervnc' -%} + tigervnc-standalone-server + {%- elif vnc_flavor|lower == 'tightvnc' and ansible_facts.os_family == 'RedHat' -%} + tightvnc-server + {%- else -%} + tightvnc-standalone-server + {%- endif %} + xorg_dev_packages: >- + {{ + ['pkgconfig', 'xorg-x11-server-devel', 'xorg-x11-util-macros', 'libglvnd-devel'] + if ansible_facts.os_family == 'RedHat' + else + ( + ['pkg-config', 'xserver-xorg-dev', 'libx11-dev', 'libxext-dev', 'libglvnd-dev', 'build-essential'] + if ansible_facts.distribution == 'Debian' + else + ['pkg-config', 'xserver-xorg-dev', 'libx11-dev', 'libxext-dev', 'libglvnd-dev'] + ) + }} + +- name: Update repository cache & install EPEL (RedHat only) + ansible.builtin.package: + name: epel-release + state: present + update_cache: yes + when: ansible_facts.os_family == 'RedHat' + retries: 10 + delay: 3 + +- name: Install desktop prerequisites & VNC package + ansible.builtin.package: + name: "{{ gui_packages + [vnc_pkg] + xorg_dev_packages }}" + state: present + retries: 10 + delay: 3 + +- name: Install kernel headers for running kernel (Debian only) + ansible.builtin.package: + name: "linux-headers-{{ ansible_kernel }}" + state: present + when: ansible_facts.distribution == 'Debian' + retries: 10 + delay: 3 + +- name: Ensure requests package is installed + pip: + name: requests + state: present + +- name: Ensure Python & tooling are installed + ansible.builtin.package: + name: + - python3 + - python3-pip + - jq + state: present + +# GPU / vWS driver setup +- name: Provision GPU setup + ansible.builtin.include_tasks: gpu_driver_setup.yaml + when: nvidia_gpu_present.rc == 0 diff --git a/community/modules/scripts/vdi-setup/roles/secret_manager/defaults/main.yaml b/community/modules/scripts/vdi-setup/roles/secret_manager/defaults/main.yaml new file mode 100644 index 0000000000..a5660000c6 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/secret_manager/defaults/main.yaml @@ -0,0 +1,16 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +secret_project: "{{ project_id }}" \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml new file mode 100644 index 0000000000..50eab08ffa --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml @@ -0,0 +1,102 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# General secret handling +- name: Generate random database password + ansible.builtin.set_fact: + database_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}" + +- name: Generate random webapp admin password + ansible.builtin.set_fact: + webapp_admin_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}" + +- name: Compute Webapp admin password hash to save to Webapp server + ansible.builtin.set_fact: + webapp_admin_hash: "{{ webapp_admin_password | hash('sha256') | upper }}" + +- name: Get access token for Secret Manager API + ansible.builtin.uri: + url: "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" + method: GET + headers: + Metadata-Flavor: Google + return_content: true + register: access_token_result + +- name: Create a GCP secret for webapp server login + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets?secretId=webapp-server-password-{{ deployment_name }}" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + replication: + automatic: {} + status_code: [200, 409] # 409 = already exists + register: webapp_secret_create + +- name: Add webapp password version to secret + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/webapp-server-password-{{ deployment_name }}:addVersion" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + payload: + data: "{{ webapp_admin_password | b64encode }}" + register: webapp_secret + +- name: Create a GCP secret for database password + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets?secretId=db-password-{{ deployment_name }}" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + replication: + automatic: {} + status_code: [200, 409] # 409 = already exists + register: database_secret_create + +- name: Add database password version to secret + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/db-password-{{ deployment_name }}:addVersion" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + payload: + data: "{{ database_password | b64encode }}" + register: database_secret + +# Per-user secret handling +- name: Generate per-user secrets and enrich list + include_tasks: user_secret_tasks.yaml + loop: "{{ vdi_users }}" + loop_control: + loop_var: item + +- name: Set enriched vdi_users_updated variable + ansible.builtin.set_fact: + vdi_users_updated: "{{ vdi_users_updated }}" + when: vdi_users_updated is defined diff --git a/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml b/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml new file mode 100644 index 0000000000..00e7fe540d --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml @@ -0,0 +1,127 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +- name: Compute default secret_name if none provided + set_fact: + secret_name: "vdi-user-password-{{ item.username }}-{{ deployment_name }}" + when: + - item.password is not defined + - item.secret_name is not defined + +- name: Determine password source for user {{ item.username }} + ansible.builtin.set_fact: + password_source: >- + {%- if item.secret_name is defined -%}secret + {%- elif item.password is defined -%}password + {%- else -%}generate{%- endif -%} + +- name: Fetch existing Secret Manager password for user {{ item.username }} + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets/{{ item.secret_name }}/versions/latest:access" + method: GET + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + return_content: true + register: sm_result + failed_when: false + when: password_source | trim == "secret" + +- name: Set VDI user password from Secret Manager + ansible.builtin.set_fact: + vdiuser_password: "{{ sm_result.json.payload.data | b64decode }}" + when: password_source == "secret" and sm_result.status == 200 + +- name: Generate random password for user {{ item.username }} + ansible.builtin.set_fact: + vdiuser_password: "{{ lookup('password', '/dev/null length=16 chars=ascii_letters,digits') }}" + when: password_source == "generate" + +- name: Set user password from provided value + ansible.builtin.set_fact: + vdiuser_password: "{{ item.password }}" + when: password_source == "password" + +- name: Create secret for user {{ item.username }} (provided password) + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets?secretId=vdi-user-password-{{ item.username }}-{{ deployment_name }}" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + replication: + automatic: {} + status_code: [200, 409] # 409 = already exists + when: password_source == "password" + +- name: Store provided password in Secret Manager for user {{ item.username }} + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets/vdi-user-password-{{ item.username }}-{{ deployment_name }}:addVersion" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + payload: + data: "{{ item.password | b64encode }}" + when: password_source == "password" + +- name: Create secret for user {{ item.username }} + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets?secretId={{ item.secret_name | default(secret_name) }}" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + replication: + automatic: {} + status_code: [200, 409] # 409 = already exists + when: password_source == "generate" + +- name: Store generated password in Secret Manager for user {{ item.username }} + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets/{{ item.secret_name | default(secret_name) }}:addVersion" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + payload: + data: "{{ vdiuser_password | b64encode }}" + when: password_source == "generate" + +- name: Build enriched user list + ansible.builtin.set_fact: + vdi_users_updated: >- + {{ + (vdi_users_updated | default([])) + + [ + item + | combine({ 'password': vdiuser_password }) + | combine( + (vnc_flavor | lower in ['tightvnc', 'tigervnc']) + | ternary({ + 'vncserver_password': lookup('community.general.random_string', length=8, special=false), + 'display_number': item.port | int - 5900 + }, {}) + ) + ] + }} diff --git a/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml new file mode 100644 index 0000000000..39ddf37c18 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml @@ -0,0 +1,51 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +- name: Provision VDI users (custom or generated passwords) + block: + + # Create the Linux user with a hashed password + - name: Create VDI user {{ item.username }} + ansible.builtin.user: + name: "{{ item.username }}" + groups: "{{ vdi_user_group }},wheel" + shell: /bin/bash + create_home: yes + append: true + password: "{{ item.password | password_hash('sha512') }}" + update_password: on_create # sets pw if user is newly created + + # SSH key setup + - name: Ensure .ssh directory for {{ item.username }} + ansible.builtin.file: + path: "/home/{{ item.username }}/.ssh" + state: directory + owner: "{{ item.username }}" + group: "{{ vdi_user_group }}" + mode: '0700' + + - name: Generate SSH key pair for {{ item.username }} + community.crypto.openssh_keypair: + path: "/home/{{ item.username }}/.ssh/id_rsa" + owner: "{{ item.username }}" + group: "{{ vdi_user_group }}" + mode: '0600' + register: ssh_key + + - name: Add public key to authorized_keys for {{ item.username }} + ansible.builtin.authorized_key: + user: "{{ item.username }}" + key: "{{ ssh_key.public_key }}" + state: present diff --git a/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml new file mode 100644 index 0000000000..569db3dd6c --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml @@ -0,0 +1,40 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +- name: Ensure 'wheel' group is present + group: + name: wheel + state: present + +- name: Ensure VDI user group exists + ansible.builtin.group: + name: "{{ vdi_user_group }}" + state: present + +# Run localā€user provisioning if selected +- name: Provision local users + ansible.builtin.include_tasks: local_users.yaml + loop: "{{ vdi_users_updated }}" + loop_control: + loop_var: item + when: user_provision | lower == 'local_users' + +# Run OS Login tasks if selected +- name: Provision OS Login users + ansible.builtin.include_tasks: os_login.yaml + loop: "{{ vdi_users_updated }}" + loop_control: + loop_var: item + when: user_provision | lower == 'os_login' diff --git a/community/modules/scripts/vdi-setup/roles/user_provision/tasks/os_login.yaml b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/os_login.yaml new file mode 100644 index 0000000000..70cb580481 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/os_login.yaml @@ -0,0 +1,15 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml new file mode 100644 index 0000000000..4bb770a0c1 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml @@ -0,0 +1,212 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +- name: Ensure PostgreSQL data & init directories exist + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: '0755' + loop: + - /opt/guacamole-db/data + - /opt/guacamole-db/initdb + +- name: Check Docker service status + ansible.builtin.service_facts: + +- name: Ensure Docker service is running + ansible.builtin.service: + name: docker + state: started + enabled: yes + register: docker_service_status + +- name: Wait for Docker daemon to be ready + ansible.builtin.wait_for: + path: /var/run/docker.sock + timeout: 30 + +- name: Test Docker connectivity + ansible.builtin.command: docker version + register: docker_test + retries: 5 + delay: 2 + until: docker_test.rc == 0 + +- name: Enable pgcrypto extension for initdb + ansible.builtin.copy: + dest: /opt/guacamole-db/initdb/00-enable-pgcrypto.sql + owner: root + group: root + mode: '0644' + content: | + -- Make digest(), hmac(), etc. available + CREATE EXTENSION IF NOT EXISTS pgcrypto; + +- name: Template per-user Guacamole user SQL + ansible.builtin.template: + src: user_bootstrap.sql.j2 + dest: "/opt/guacamole-db/initdb/02-{{ item.username }}_user.sql" + owner: root + group: root + mode: '0644' + loop: "{{ vdi_users_updated }}" + loop_control: + loop_var: item + vars: + username: "{{ item.username }}" + vdi_password: "{{ item.password }}" + +- name: Template per-user SQL for Guacamole connections + ansible.builtin.template: + src: connection_bootstrap.sql.j2 + dest: "/opt/guacamole-db/initdb/03-{{ item.username }}_connection.sql" + owner: root + group: root + mode: '0644' + loop: "{{ vdi_users_updated }}" + loop_control: + loop_var: item + vars: + port: "{{ item.port }}" + username: "{{ item.username }}" + vnc_password: "{{ item.vncserver_password }}" + +- name: Generate combined initdb.sql + ansible.builtin.shell: | + docker run --rm \ + -v /opt/guacamole-db/initdb:/opt/guacamole-db/initdb \ + guacamole/guacamole:latest \ + /opt/guacamole/bin/initdb.sh --postgresql \ + > /opt/guacamole-db/initdb/01-initdb.sql + args: + creates: /opt/guacamole-db/initdb/01-initdb.sql + +- name: Update guacadmin password hash in 01-initdb.sql + ansible.builtin.replace: + path: /opt/guacamole-db/initdb/01-initdb.sql + regexp: "decode\\('CA458A7D494E3BE824F5E1E175A1556C0F8EEF2C2D7DF3633BEC4A29C4411960', 'hex'\\)" + replace: "decode('{{ webapp_admin_hash }}', 'hex')" + +- name: Remove salt from guacadmin in 01-initdb.sql + ansible.builtin.replace: + path: /opt/guacamole-db/initdb/01-initdb.sql + regexp: "decode\\('FE24ADC5E11E2B25288D1704ABE67A79E342ECC26064CE69C5B3177795A82264', 'hex'\\)" + replace: "NULL" + +- name: Pull required Docker images + ansible.builtin.command: docker pull {{ item }} + loop: + - postgres:latest + - guacamole/guacd:latest + - guacamole/guacamole:latest + +- name: Create a dedicated Docker network + ansible.builtin.command: docker network create guac_net + register: network_create + failed_when: + - network_create.rc != 0 + - "'already exists' not in network_create.stderr" + +- name: Start Guacamole PostgreSQL container + ansible.builtin.command: > + docker run -d --name guac_db --network guac_net + -p 5432:5432 + -e POSTGRES_USER=guacamole_db + -e POSTGRES_PASSWORD={{ database_password }} + -e POSTGRES_DB=guacamole_db + -v /opt/guacamole-db/data:/var/lib/postgresql/data + -v /opt/guacamole-db/initdb:/docker-entrypoint-initdb.d + --restart always + postgres:latest + register: postgres_start + failed_when: + - postgres_start.rc != 0 + - "'already in use' not in postgres_start.stderr" + +- name: Wait for PostgreSQL on host port + ansible.builtin.wait_for: + host: 127.0.0.1 + port: 5432 + delay: 2 + timeout: 60 + +- name: Start guacd container + ansible.builtin.command: > + docker run -d --name guacd --network host + --restart always + guacamole/guacd:latest + register: guacd_start + failed_when: + - guacd_start.rc != 0 + - "'already in use' not in guacd_start.stderr" + +- name: Start Guacamole webapp container + ansible.builtin.command: > + docker run -d --name guac_app --network host + -e POSTGRES_HOSTNAME=127.0.0.1 + -e POSTGRES_PORT=5432 + -e POSTGRES_DATABASE=guacamole_db + -e POSTGRES_USER=guacamole_db + -e POSTGRES_PASSWORD={{ database_password }} + -e POSTGRES_AUTO_CREATE_ACCOUNTS=true + -e GUACD_HOSTNAME=localhost + -e GUACD_PORT=4822 + --restart always + guacamole/guacamole:latest + register: guac_app_start + failed_when: + - guac_app_start.rc != 0 + - "'already in use' not in guac_app_start.stderr" + +- name: Wait for Guacamole HTTP endpoint + ansible.builtin.wait_for: + host: 127.0.0.1 + port: "{{ vdi_webapp_port }}" + delay: 2 + timeout: 60 + +- name: Check Guacamole UI is up + ansible.builtin.uri: + url: "http://localhost:{{ vdi_webapp_port }}/guacamole/" + status_code: 200 + return_content: no + register: guac_health + until: guac_health.status == 200 + retries: 10 + delay: 3 + +- name: Obtain Guacamole API token to verify login + ansible.builtin.uri: + url: "http://localhost:{{ vdi_webapp_port }}/guacamole/api/tokens" + method: POST + body_format: form-urlencoded + body: + username: "guacadmin" + password: "{{ webapp_admin_password }}" + return_content: yes + register: guac_api + until: + - guac_api.status == 200 + - guac_api.json.authToken is defined + retries: 10 + delay: 3 + +- name: Remove bootstrap SQL files unless debug is enabled + ansible.builtin.file: + path: /opt/guacamole-db/initdb + state: absent + when: not debug | default(false) diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml new file mode 100644 index 0000000000..4257eb3b52 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml @@ -0,0 +1,26 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +- name: "Install Guacamole" + include_tasks: guacamole.yaml + when: vdi_tool | lower == 'guacamole' + +- name: "Install NoMachine" + include_tasks: nomachine.yaml + when: vdi_tool | lower == 'nomachine' + +- name: "Install Workspot" + include_tasks: workspot.yaml + when: vdi_tool | lower == 'workspot' diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/nomachine.yaml b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/nomachine.yaml new file mode 100644 index 0000000000..70cb580481 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/nomachine.yaml @@ -0,0 +1,15 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/workspot.yaml b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/workspot.yaml new file mode 100644 index 0000000000..70cb580481 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/workspot.yaml @@ -0,0 +1,15 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/connection_bootstrap.sql.j2 b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/connection_bootstrap.sql.j2 new file mode 100644 index 0000000000..e90562fccc --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/connection_bootstrap.sql.j2 @@ -0,0 +1,72 @@ +-- VNC Connection for {{ username }} +WITH + user_entity AS ( + SELECT entity_id + FROM guacamole_entity + WHERE name = '{{ username }}' AND type = 'USER' + ), + new_connection AS ( + INSERT INTO guacamole_connection (connection_name, protocol) + VALUES ('{{ username }}', 'vnc') + RETURNING connection_id + ), + insert_params AS ( + INSERT INTO guacamole_connection_parameter (connection_id, parameter_name, parameter_value) + SELECT connection_id, param_name, param_value + FROM new_connection, (VALUES + ('hostname', '{{ ansible_default_ipv4.address }}'), + ('port', '{{ port }}'), + ('username', '{{ username }}'), + ('password', '{{ vnc_password }}') + ) AS params(param_name, param_value) + ), + insert_permissions AS ( + INSERT INTO guacamole_connection_permission (entity_id, connection_id, permission) + SELECT + user_entity.entity_id, + new_connection.connection_id, + perms.permission::guacamole_object_permission_type + FROM user_entity, new_connection, + (VALUES ('READ'),('UPDATE'),('DELETE'),('ADMINISTER')) AS perms(permission) + ) +SELECT 1; + +-- SSH Connection for {{ username }} +WITH + user_entity AS ( + SELECT entity_id + FROM guacamole_entity + WHERE name = '{{ username }}' AND type = 'USER' + ), + new_connection_ssh AS ( + INSERT INTO guacamole_connection (connection_name, protocol) + VALUES ('{{ username }} SSH', 'ssh') + RETURNING connection_id + ), + insert_params_ssh AS ( + INSERT INTO guacamole_connection_parameter (connection_id, parameter_name, parameter_value) + SELECT connection_id, param_name, param_value + FROM new_connection_ssh, (VALUES + ('hostname', '{{ ansible_default_ipv4.address }}'), + ('port', '22'), + ('username', '{{ username }}'), + ( + 'private-key', + E'{{ lookup("file", "/home/" ~ username ~ "/.ssh/id_rsa") + | replace("\\", "\\\\") + | replace("'", "''") + | replace("\r", "") + | replace("\n", "\\n") }}' + ) + ) AS params(param_name, param_value) + ), + insert_permissions_ssh AS ( + INSERT INTO guacamole_connection_permission (entity_id, connection_id, permission) + SELECT + user_entity.entity_id, + new_connection_ssh.connection_id, + perms.permission::guacamole_object_permission_type + FROM user_entity, new_connection_ssh, + (VALUES ('READ'),('UPDATE'),('DELETE'),('ADMINISTER')) AS perms(permission) + ) +SELECT 1; diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/user_bootstrap.sql.j2 b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/user_bootstrap.sql.j2 new file mode 100644 index 0000000000..e9c7e41c37 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/user_bootstrap.sql.j2 @@ -0,0 +1,22 @@ +-- Create Guacamole entity + user for {{ username }} +WITH new_entity AS ( + INSERT INTO guacamole_entity (name, type) + VALUES ('{{ username }}', 'USER') + RETURNING entity_id +) +INSERT INTO guacamole_user ( + entity_id, + password_salt, + password_hash, + password_date, + disabled, + expired +) +SELECT + entity_id, + NULL, + digest('{{ vdi_password }}', 'SHA-256'), + now(), + false, + false +FROM new_entity; \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml new file mode 100644 index 0000000000..bc81213782 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml @@ -0,0 +1,23 @@ + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +- name: "Install TigerVNC" + include_tasks: tigervnc.yaml + when: vnc_flavor | lower == 'tigervnc' + +- name: "Install TightVNC" + include_tasks: tightvnc.yaml + when: vnc_flavor | lower == 'tightvnc' diff --git a/community/modules/scripts/vdi-setup/roles/vnc/tasks/tigervnc.yaml b/community/modules/scripts/vdi-setup/roles/vnc/tasks/tigervnc.yaml new file mode 100644 index 0000000000..0401806493 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vnc/tasks/tigervnc.yaml @@ -0,0 +1,69 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# Per-user configuration +- name: Configure per-user VNC settings + include_tasks: vnc_user_config.yaml + loop: "{{ vdi_users_updated }}" + loop_control: + loop_var: vdi_user + +# Global VNC users mapping (for systemd service) +- name: Template TigerVNC users file + ansible.builtin.template: + src: vncserver.users.j2 + dest: /etc/tigervnc/vncserver.users + mode: '0644' + vars: + guac_map: "{{ vdi_users_updated }}" + +# Systemd units +- name: Deploy per-user TigerVNC systemd unit (Ubuntu/Debian) + ansible.builtin.template: + src: vncserver@.service.ubuntu.j2 + dest: "/etc/systemd/system/vncserver@{{ item.username }}.service" + mode: '0644' + vars: + display_number: "{{ item.display_number }}" + when: ansible_distribution in ["Ubuntu", "Debian"] + loop: "{{ vdi_users_updated }}" + +- name: Deploy TigerVNC systemd unit (Rocky) + ansible.builtin.template: + src: vncserver@.service.rocky.j2 + dest: /etc/systemd/system/vncserver@.service + mode: '0644' + when: ansible_os_family == "RedHat" + +- name: Reload systemd daemon + ansible.builtin.systemd: + daemon_reload: yes + +# Enable & start each user's VNC server service +- name: Enable & start TigerVNC (Ubuntu/Debian) + systemd: + name: "vncserver@{{ item.username }}" + enabled: yes + state: started + when: ansible_distribution in ["Ubuntu", "Debian"] + loop: "{{ vdi_users_updated }}" + +- name: Enable & start TigerVNC (Rocky) + systemd: + name: "vncserver@:{{ item.display_number }}" + enabled: yes + state: started + when: ansible_os_family == "RedHat" + loop: "{{ vdi_users_updated }}" diff --git a/community/modules/scripts/vdi-setup/roles/vnc/tasks/tightvnc.yaml b/community/modules/scripts/vdi-setup/roles/vnc/tasks/tightvnc.yaml new file mode 100644 index 0000000000..70cb580481 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vnc/tasks/tightvnc.yaml @@ -0,0 +1,15 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + diff --git a/community/modules/scripts/vdi-setup/roles/vnc/tasks/vnc_user_config.yaml b/community/modules/scripts/vdi-setup/roles/vnc/tasks/vnc_user_config.yaml new file mode 100644 index 0000000000..206551b270 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vnc/tasks/vnc_user_config.yaml @@ -0,0 +1,59 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +- name: Ensure VNC configuration directory exists + ansible.builtin.file: + path: "/home/{{ vdi_user.username }}/.vnc" + state: directory + owner: "{{ vdi_user.username }}" + group: "{{ vdi_user_group }}" + mode: '0700' + +- name: Write VNC xstartup for {{ vdi_user.username }} (Ubuntu/Debian) + ansible.builtin.copy: + dest: "/home/{{ vdi_user.username }}/.vnc/xstartup" + content: | + #!/bin/sh + unset SESSION_MANAGER + unset DBUS_SESSION_BUS_ADDRESS + exec startxfce4 + owner: "{{ vdi_user.username }}" + group: "{{ vdi_user_group }}" + mode: '0755' + when: ansible_distribution in ["Ubuntu", "Debian"] + +- name: Write VNC password file for {{ vdi_user.username }} + ansible.builtin.shell: | + echo "{{ vdi_user.vncserver_password }}" | vncpasswd -f > "/home/{{ vdi_user.username }}/.vnc/passwd" + args: + creates: "/home/{{ vdi_user.username }}/.vnc/passwd" + become_user: "{{ vdi_user.username }}" + +- name: Fix passwd file permissions + ansible.builtin.file: + path: "/home/{{ vdi_user.username }}/.vnc/passwd" + owner: "{{ vdi_user.username }}" + group: "{{ vdi_user_group }}" + mode: '0600' + +- name: Copy per-user VNC config + ansible.builtin.template: + src: config.j2 + dest: "/home/{{ vdi_user.username }}/.vnc/config" + owner: "{{ vdi_user.username }}" + group: "{{ vdi_user_group }}" + mode: '0644' + vars: + vdi_resolution: "{{ vdi_resolution }}" diff --git a/community/modules/scripts/vdi-setup/roles/vnc/templates/config.j2 b/community/modules/scripts/vdi-setup/roles/vnc/templates/config.j2 new file mode 100644 index 0000000000..605ec72eca --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vnc/templates/config.j2 @@ -0,0 +1,2 @@ +session=xfce +geometry={{ vdi_resolution }} diff --git a/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver.users.j2 b/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver.users.j2 new file mode 100644 index 0000000000..cbdab2dbc0 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver.users.j2 @@ -0,0 +1,3 @@ +{% for user in guac_map %} +:{{ user.port - 5900 }}={{ user.username }} +{% endfor %} diff --git a/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.rocky.j2 b/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.rocky.j2 new file mode 100644 index 0000000000..a7cc3cc10a --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.rocky.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=TigerVNC server for user %i +After=syslog.target network.target + +[Service] +Type=forking +ExecStart=/usr/libexec/vncsession-start %i +ExecStop=-/bin/true +PIDFile=/run/vncsession-%i.pid + +Restart=always +RestartSec=15 + +[Install] +WantedBy=multi-user.target diff --git a/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.ubuntu.j2 b/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.ubuntu.j2 new file mode 100644 index 0000000000..1df046f446 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.ubuntu.j2 @@ -0,0 +1,16 @@ +[Unit] +Description=TigerVNC server for user %i +After=network.target + +[Service] +Type=forking +User=%i +Environment=HOME=/home/%i +ExecStart=/usr/bin/vncserver -localhost no :{{ display_number }} +ExecStop=/usr/bin/vncserver -kill :{{ display_number }} + +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl b/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl new file mode 100644 index 0000000000..15bd97d02e --- /dev/null +++ b/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl @@ -0,0 +1,12 @@ +# Generated by Cluster Toolkit + +- hosts: localhost + become: true + vars_files: + - "/tmp/vdi/vars.yaml" + tasks: + +%{ for r in roles } + - import_role: + name: "${r}" +%{ endfor } \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl b/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl new file mode 100644 index 0000000000..b393203a62 --- /dev/null +++ b/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl @@ -0,0 +1,23 @@ +# Generated by Cluster Toolkit + +deployment_name: ${deployment_name} +project_id: ${project_id} +user_provision: ${user_provision} +vnc_flavor: ${vnc_flavor} +vdi_tool: ${vdi_tool} +vdi_user_group: ${vdi_user_group} +vdi_webapp_port: ${vdi_webapp_port} +vdi_resolution: ${vdi_resolution} + +vdi_users: +%{ for user in vdi_users } + - username: ${user.username} + port: ${user.port} +%{ if user.secret_name != null } + secret_name: "${user.secret_name}" +%{ endif } +%{ if user.secret_project != null } + secret_project: "${user.secret_project}" +%{ endif } + +%{ endfor } \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/validation.tf b/community/modules/scripts/vdi-setup/validation.tf new file mode 100644 index 0000000000..64fa503745 --- /dev/null +++ b/community/modules/scripts/vdi-setup/validation.tf @@ -0,0 +1,53 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "terraform_data" "input_validation" { + lifecycle { + precondition { + condition = contains(["tigervnc"], var.vnc_flavor) + error_message = "vnc_flavor must be tigervnc." + } + + precondition { + condition = contains(["guacamole"], var.vdi_tool) + error_message = "vdi_tool must be one of: guacamole." + } + + precondition { + condition = contains(["local_users"], var.user_provision) + error_message = "user_provision must be local_users." + } + + precondition { + condition = can(regex("^[1-9][0-9]*x[1-9][0-9]*$", var.vdi_resolution)) + error_message = "vdi_resolution must be in the form WIDTHxHEIGHT (e.g. 1920x1080)." + } + + precondition { + condition = var.user_provision == "local_users" || length(var.vdi_users) == 0 + error_message = "vdi_users may only be set when user_provision = local_users." + } + + precondition { + condition = ( + var.vnc_flavor != "tigervnc" && var.vnc_flavor != "tightvnc" + ) || alltrue([ + for user in var.vdi_users : ( + user.port >= var.vnc_port_min && user.port <= var.vnc_port_max + ) + ]) + error_message = "Each VDI user must have a port between 5901 and 5999 when VNC is used." + } + } +} diff --git a/community/modules/scripts/vdi-setup/variables.tf b/community/modules/scripts/vdi-setup/variables.tf new file mode 100644 index 0000000000..9b1ce654ab --- /dev/null +++ b/community/modules/scripts/vdi-setup/variables.tf @@ -0,0 +1,103 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "Project in which the HPC deployment will be created." + type = string +} + +variable "deployment_name" { + description = "The name of the deployment." + type = string +} + +variable "region" { + description = "Region to place bucket containing startup script." + type = string +} + +variable "labels" { + description = "Key-value pairs of labels to be added to created resources." + type = map(string) +} + +variable "vnc_flavor" { + description = "The VNC server flavor to use (tigervnc currently supported)" + type = string + default = "tigervnc" +} + +variable "vdi_tool" { + type = string + description = "VDI tool to deploy (guacamole currently supported)." + default = "guacamole" +} + +variable "user_provision" { + type = string + description = "User type to create (local_users supported. os-login to do." + default = "local_users" +} + +variable "vdi_user_group" { + type = string + description = "Unix group to create/use for VDI users." + default = "vdiusers" +} + +variable "vdi_resolution" { + type = string + description = "Desktop resolution for VNC sessions (e.g. 1920x1080)." + default = "1920x1080" +} + +variable "vdi_webapp_port" { + type = string + description = "Port to serve the Webapp interface from if applicable" + default = "8080" +} + +variable "vdi_users" { + description = "List of VDI users to configure. Passwords are handled securely by the Ansible roles: if secret_name is provided, the password is fetched from Secret Manager; if neither password nor secret_name is provided, a random password is generated and stored in Secret Manager. If secret_project is provided, it specifies the GCP project where the secret is stored (defaults to the deployment project)." + type = list(object({ + username = string + port = number + secret_name = optional(string) + secret_project = optional(string) + })) + default = [] +} + +variable "vnc_port_min" { + type = number + default = 5901 + description = "Minimum valid VNC port." +} +variable "vnc_port_max" { + type = number + default = 5999 + description = "Maximum valid VNC port." +} + +variable "vdi_instance_ip" { + description = "The IP address of the VDI instance" + type = string + default = null +} + +variable "vdi_instance_name" { + description = "The name of the VDI instance" + type = string + default = null +} From 1db8b78f0bb8c641a1c31b3f7e0eb38f070176c0 Mon Sep 17 00:00:00 2001 From: Scott Gordon Date: Wed, 16 Jul 2025 15:23:42 +0100 Subject: [PATCH 02/21] docker install fix for rocky --- .../startup-script/files/install_docker.yml | 60 ++++++++++++++++++- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/modules/scripts/startup-script/files/install_docker.yml b/modules/scripts/startup-script/files/install_docker.yml index f9b0abeb14..af251014b8 100644 --- a/modules/scripts/startup-script/files/install_docker.yml +++ b/modules/scripts/startup-script/files/install_docker.yml @@ -20,24 +20,75 @@ docker_data_root: '' docker_daemon_config: '' enable_docker_world_writable: false + is_rocky_linux: "{{ ansible_distribution in ['Rocky', 'Rocky Linux'] }}" tasks: + - name: Gather distribution facts + ansible.builtin.setup: + gather_subset: platform - name: Check if docker is installed ansible.builtin.stat: path: /usr/bin/docker register: docker_binary - - name: Download Docker Installer + - name: Add Docker GPG key (Rocky) + ansible.builtin.rpm_key: + state: present + key: https://download.docker.com/linux/rhel/gpg + when: + - is_rocky_linux + - not docker_binary.stat.exists + register: add_docker_key + retries: 30 + delay: 10 + until: add_docker_key is succeeded + - name: Add Docker repository for Rocky Linux + ansible.builtin.command: > + dnf config-manager --add-repo + https://download.docker.com/linux/rhel/docker-ce.repo + when: + - is_rocky_linux + - not docker_binary.stat.exists + notify: Refresh dnf cache + - name: Install Docker Engine and related packages on Rocky + ansible.builtin.package: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + state: present + update_cache: yes + when: + - is_rocky_linux + - not docker_binary.stat.exists + - name: Ensure Docker SDK for Python is installed + block: + - name: Install python3-pip + ansible.builtin.package: + name: python3-pip + state: present + - name: Install Docker SDK for Python + ansible.builtin.pip: + name: docker + executable: /usr/local/ghpc-venv/bin/pip3 + when: is_rocky_linux + - name: Download Docker Installer (get.docker.com) ansible.builtin.get_url: url: https://get.docker.com dest: /tmp/get-docker.sh owner: root group: root mode: '0644' - when: not docker_binary.stat.exists + when: + - not is_rocky_linux + - not docker_binary.stat.exists - name: Install Docker ansible.builtin.command: sh /tmp/get-docker.sh register: docker_installed changed_when: docker_installed.rc != 0 - when: not docker_binary.stat.exists + when: + - not is_rocky_linux + - not docker_binary.stat.exists - name: Create Docker daemon configuration ansible.builtin.copy: dest: /etc/docker/daemon.json @@ -104,6 +155,9 @@ ansible.builtin.service: name: docker.service state: restarted + - name: Refresh dnf cache + ansible.builtin.dnf: + update_cache: yes post_tasks: - name: Start Docker From 6751e70f506d76ba92677d5b40cc1bfca1384d6c Mon Sep 17 00:00:00 2001 From: Scott Gordon Date: Thu, 17 Jul 2025 10:37:15 +0100 Subject: [PATCH 03/21] pre-commit tweaks --- community/modules/scripts/vdi-setup/README.md | 66 ++++++++++++++++++- community/modules/scripts/vdi-setup/main.tf | 19 +++--- .../modules/scripts/vdi-setup/outputs.tf | 6 +- .../roles/base_os/tasks/gpu_driver_setup.yaml | 2 +- .../roles/secret_manager/defaults/main.yaml | 2 +- .../roles/user_provision/tasks/os_login.yaml | 1 - .../roles/vdi_tool/tasks/nomachine.yaml | 1 - .../roles/vdi_tool/tasks/workspot.yaml | 1 - .../vdi_tool/templates/user_bootstrap.sql.j2 | 2 +- .../vdi-setup/roles/vnc/tasks/tightvnc.yaml | 1 - .../vdi-setup/templates/install.yaml.tftpl | 2 +- .../vdi-setup/templates/vars.yaml.tftpl | 2 +- .../modules/scripts/vdi-setup/validation.tf | 8 +-- 13 files changed, 83 insertions(+), 30 deletions(-) diff --git a/community/modules/scripts/vdi-setup/README.md b/community/modules/scripts/vdi-setup/README.md index b4dc34a00f..c7389e8305 100644 --- a/community/modules/scripts/vdi-setup/README.md +++ b/community/modules/scripts/vdi-setup/README.md @@ -14,8 +14,7 @@ ## Description -Mostly tested with Rocky. Working on Debian and Ubuntu images (commented in blueprint example below). -Distro / VDI 'flavour' variations are handled in the 'base_os' role for the most part. +Creates a containerised Guacamole instance. Works with the Debian and Ubuntu images shown in blueprint example below. ### Service Startup and Health Checking @@ -26,7 +25,7 @@ The VDI module includes basic health checks to ensure services start properly: The VDI module supports flexible Secret Manager integration: - **Default Behavior**: Secrets are stored in the deployment project -- **Cross-Project Secrets**: Use `secret_project` to specify a different GCP project for user secrets +- **Cross-Project Secrets**: Use `secret_project` to specify a different GCP project if providing user secrets - **Automatic Password Generation**: If no `secret_name` is provided, random passwords are generated and stored - **Existing Secret Retrieval**: Provide `secret_name` to use existing secrets from Secret Manager @@ -162,3 +161,64 @@ After deployment, you can access the VDI in several ways: --zone=us-central1-a ``` - Guacamole will then be accessible from http://localhost:8080/guacamole/ + + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [archive](#provider\_archive) | n/a | +| [google](#provider\_google) | n/a | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | +| [terraform_data.input_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [archive_file.roles_tar](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | n/a | yes | +| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | +| [user\_provision](#input\_user\_provision) | User type to create (local\_users supported. os-login to do. | `string` | `"local_users"` | no | +| [vdi\_instance\_ip](#input\_vdi\_instance\_ip) | The IP address of the VDI instance | `string` | `null` | no | +| [vdi\_instance\_name](#input\_vdi\_instance\_name) | The name of the VDI instance | `string` | `null` | no | +| [vdi\_resolution](#input\_vdi\_resolution) | Desktop resolution for VNC sessions (e.g. 1920x1080). | `string` | `"1920x1080"` | no | +| [vdi\_tool](#input\_vdi\_tool) | VDI tool to deploy (guacamole currently supported). | `string` | `"guacamole"` | no | +| [vdi\_user\_group](#input\_vdi\_user\_group) | Unix group to create/use for VDI users. | `string` | `"vdiusers"` | no | +| [vdi\_users](#input\_vdi\_users) | List of VDI users to configure. Passwords are handled securely by the Ansible roles: if secret\_name is provided, the password is fetched from Secret Manager; if neither password nor secret\_name is provided, a random password is generated and stored in Secret Manager. If secret\_project is provided, it specifies the GCP project where the secret is stored (defaults to the deployment project). | 
list(object({
    username       = string
    port           = number
    secret_name    = optional(string)
    secret_project = optional(string)
  }))
| `[]` | no | +| [vdi\_webapp\_port](#input\_vdi\_webapp\_port) | Port to serve the Webapp interface from if applicable | `string` | `"8080"` | no | +| [vnc\_flavor](#input\_vnc\_flavor) | The VNC server flavor to use (tigervnc currently supported) | `string` | `"tigervnc"` | no | +| [vnc\_port\_max](#input\_vnc\_port\_max) | Maximum valid VNC port. | `number` | `5999` | no | +| [vnc\_port\_min](#input\_vnc\_port\_min) | Minimum valid VNC port. | `number` | `5901` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [guacamole\_admin\_password\_secret](#output\_guacamole\_admin\_password\_secret) | The name of the Secret Manager secret containing the Guacamole admin password | +| [guacamole\_admin\_username](#output\_guacamole\_admin\_username) | The admin username for Guacamole | +| [guacamole\_url](#output\_guacamole\_url) | The URL to access the Guacamole web interface | +| [startup\_script](#output\_startup\_script) | Combined startup script that installs VDI (VNC, Guacamole, users). | +| [vdi\_instance\_ip](#output\_vdi\_instance\_ip) | The IP address of the VDI instance | +| [vdi\_instance\_name](#output\_vdi\_instance\_name) | The name of the VDI instance | +| [vdi\_runner](#output\_vdi\_runner) | Shell runner wrapping Ansible playbook + roles (for custom-image or direct use). | +| [vdi\_user\_credentials](#output\_vdi\_user\_credentials) | Map of VDI user credentials stored in Secret Manager | + diff --git a/community/modules/scripts/vdi-setup/main.tf b/community/modules/scripts/vdi-setup/main.tf index a432fea15a..38aec83f2c 100644 --- a/community/modules/scripts/vdi-setup/main.tf +++ b/community/modules/scripts/vdi-setup/main.tf @@ -13,6 +13,7 @@ # limitations under the License. locals { + # This label allows for billing report tracking based on module. labels = merge(var.labels, { ghpc_module = "vdi-setup", ghpc_role = "scripts" }) } @@ -80,25 +81,21 @@ locals { # Run the rendered playbook via ansible-local { - type = "ansible-local" - content = templatefile("${path.module}/templates/install.yaml.tftpl", + type = "ansible-local" + content = templatefile("${path.module}/templates/install.yaml.tftpl", { - roles = ["base_os", "secret_manager", "user_provision", "vnc", "vdi_tool"], + roles = ["base_os", "secret_manager", "user_provision", "vnc", "vdi_tool"], } ) destination = "/tmp/vdi/install.yaml" - # Todo: turn off debug and '-v' later: - args = "--extra-vars @/tmp/vdi/vars.yaml -v --extra-vars debug=true" + # Debug mode: + # args = "--extra-vars @/tmp/vdi/vars.yaml -v --extra-vars debug=true" + args = "--extra-vars @/tmp/vdi/vars.yaml" }, # Todo: another runner here to delete /tmp/vdi afterwards? ] - required_apis = [ - "compute.googleapis.com", - "secretmanager.googleapis.com", - ] - - bucket_name = "${substr(var.deployment_name,0,39)}-vdi-scripts-${substr(md5(var.deployment_name),0,8)}" + bucket_name = "${substr(var.deployment_name, 0, 39)}-vdi-scripts-${substr(md5(var.deployment_name), 0, 8)}" } # Bucket to stage runners diff --git a/community/modules/scripts/vdi-setup/outputs.tf b/community/modules/scripts/vdi-setup/outputs.tf index a38dfbf8f4..f3a9787644 100644 --- a/community/modules/scripts/vdi-setup/outputs.tf +++ b/community/modules/scripts/vdi-setup/outputs.tf @@ -41,9 +41,9 @@ output "vdi_user_credentials" { description = "Map of VDI user credentials stored in Secret Manager" value = { for user in var.vdi_users : user.username => { - username = user.username - port = user.port - secret_name = user.secret_name != null ? user.secret_name : "vdi-user-password-${user.username}-${var.deployment_name}" + username = user.username + port = user.port + secret_name = user.secret_name != null ? user.secret_name : "vdi-user-password-${user.username}-${var.deployment_name}" } } } diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml index eb2ff0ddb2..9c67c60f56 100644 --- a/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml @@ -71,4 +71,4 @@ ansible.builtin.command: cmd: /tmp/nvidia-driver.run --dkms --silent creates: /usr/bin/nvidia-smi - when: is_nvidia_vws_supported \ No newline at end of file + when: is_nvidia_vws_supported diff --git a/community/modules/scripts/vdi-setup/roles/secret_manager/defaults/main.yaml b/community/modules/scripts/vdi-setup/roles/secret_manager/defaults/main.yaml index a5660000c6..15a621b3ad 100644 --- a/community/modules/scripts/vdi-setup/roles/secret_manager/defaults/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/secret_manager/defaults/main.yaml @@ -13,4 +13,4 @@ # limitations under the License. --- -secret_project: "{{ project_id }}" \ No newline at end of file +secret_project: "{{ project_id }}" diff --git a/community/modules/scripts/vdi-setup/roles/user_provision/tasks/os_login.yaml b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/os_login.yaml index 70cb580481..3039c31d62 100644 --- a/community/modules/scripts/vdi-setup/roles/user_provision/tasks/os_login.yaml +++ b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/os_login.yaml @@ -12,4 +12,3 @@ # See the License for the specific language governing permissions and # limitations under the License. --- - diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/nomachine.yaml b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/nomachine.yaml index 70cb580481..3039c31d62 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/nomachine.yaml +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/nomachine.yaml @@ -12,4 +12,3 @@ # See the License for the specific language governing permissions and # limitations under the License. --- - diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/workspot.yaml b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/workspot.yaml index 70cb580481..3039c31d62 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/workspot.yaml +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/workspot.yaml @@ -12,4 +12,3 @@ # See the License for the specific language governing permissions and # limitations under the License. --- - diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/user_bootstrap.sql.j2 b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/user_bootstrap.sql.j2 index e9c7e41c37..fb6fc25571 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/user_bootstrap.sql.j2 +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/user_bootstrap.sql.j2 @@ -19,4 +19,4 @@ SELECT now(), false, false -FROM new_entity; \ No newline at end of file +FROM new_entity; diff --git a/community/modules/scripts/vdi-setup/roles/vnc/tasks/tightvnc.yaml b/community/modules/scripts/vdi-setup/roles/vnc/tasks/tightvnc.yaml index 70cb580481..3039c31d62 100644 --- a/community/modules/scripts/vdi-setup/roles/vnc/tasks/tightvnc.yaml +++ b/community/modules/scripts/vdi-setup/roles/vnc/tasks/tightvnc.yaml @@ -12,4 +12,3 @@ # See the License for the specific language governing permissions and # limitations under the License. --- - diff --git a/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl b/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl index 15bd97d02e..eed3983fb0 100644 --- a/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl +++ b/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl @@ -9,4 +9,4 @@ %{ for r in roles } - import_role: name: "${r}" -%{ endfor } \ No newline at end of file +%{ endfor } diff --git a/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl b/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl index b393203a62..9960ecf9db 100644 --- a/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl +++ b/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl @@ -20,4 +20,4 @@ vdi_users: secret_project: "${user.secret_project}" %{ endif } -%{ endfor } \ No newline at end of file +%{ endfor } diff --git a/community/modules/scripts/vdi-setup/validation.tf b/community/modules/scripts/vdi-setup/validation.tf index 64fa503745..02c3dfc822 100644 --- a/community/modules/scripts/vdi-setup/validation.tf +++ b/community/modules/scripts/vdi-setup/validation.tf @@ -42,10 +42,10 @@ resource "terraform_data" "input_validation" { precondition { condition = ( var.vnc_flavor != "tigervnc" && var.vnc_flavor != "tightvnc" - ) || alltrue([ - for user in var.vdi_users : ( - user.port >= var.vnc_port_min && user.port <= var.vnc_port_max - ) + ) || alltrue([ + for user in var.vdi_users : ( + user.port >= var.vnc_port_min && user.port <= var.vnc_port_max + ) ]) error_message = "Each VDI user must have a port between 5901 and 5999 when VNC is used." } From fe6d774b8c9a06765bd458ae625c597f718672e9 Mon Sep 17 00:00:00 2001 From: Scott Gordon Date: Thu, 17 Jul 2025 15:10:45 +0100 Subject: [PATCH 04/21] nvidia driver ubuntu fix --- @ | 66 +++++++++ community/modules/scripts/vdi-setup/README.md | 51 +++---- .../vdi-setup/roles/base_os/tasks/debian.yaml | 41 ++++++ .../roles/base_os/tasks/gpu_driver_setup.yaml | 41 +++--- .../tasks/gpu_driver_setup/debian.yaml | 30 ++++ .../base_os/tasks/gpu_driver_setup/rocky.yaml | 36 +++++ .../tasks/gpu_driver_setup/ubuntu.yaml | 73 +++++++++ .../vdi-setup/roles/base_os/tasks/main.yaml | 68 ++------- .../vdi-setup/roles/base_os/tasks/rocky.yaml | 42 ++++++ .../vdi-setup/roles/base_os/tasks/ubuntu.yaml | 138 ++++++++++++++++++ .../modules/scripts/vdi-setup/versions.tf | 29 ++++ 11 files changed, 506 insertions(+), 109 deletions(-) create mode 100644 @ create mode 100644 community/modules/scripts/vdi-setup/roles/base_os/tasks/debian.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/debian.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/rocky.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/ubuntu.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/base_os/tasks/rocky.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml create mode 100644 community/modules/scripts/vdi-setup/versions.tf diff --git a/@ b/@ new file mode 100644 index 0000000000..5e2392bfcd --- /dev/null +++ b/@ @@ -0,0 +1,66 @@ + +# Default compute SA will require IAM role: Secret Manager Secret Accessor + +blueprint_name: vdi-test-scott + +vars: + deployment_name: vdi-test-scott + project_id: hpc-discovery-external + region: us-central1 + zone: us-central1-a + +deployment_groups: +- group: primary + modules: + - id: network1 + source: modules/network/vpc + settings: + extra_iap_ports: [8080] + firewall_rules: + - name: allow-guacamole-8080-ext + description: Allow external ingress to Guacamole on TCP port 8080 + direction: INGRESS + ranges: ["0.0.0.0/0"] # or rev proxy address? + allow: + - protocol: tcp + ports: ["8080"] + + - id: enable-apis + source: community/modules/project/service-enablement + settings: + gcp_service_list: + - secretmanager.googleapis.com + - storage.googleapis.com + - compute.googleapis.com + + - id: vdi-setup + source: community/modules/scripts/vdi-setup + settings: + vnc_flavor: tigervnc + vdi_tool: guacamole + user_provision: local_users + vdi_user_group: vdiusers + vdi_resolution: 1920x1080 + vdi_users: + - username: alice + port: 5901 + - username: bob + port: 5902 + secret_name: a-password-for-bob + + - id: machine + source: modules/compute/vm-instance + settings: + instance_image: + #family: debian-11 + #project: debian-cloud + #family: ubuntu-2204-lts + #project: ubuntu-os-cloud + family: hpc-rocky-linux-8 + project: cloud-hpc-image-public + machine_type: g2-standard-8 + disable_public_ips: false + tags: ["guacamole"] + use: + - network1 + - vdi-setup diff --git a/community/modules/scripts/vdi-setup/README.md b/community/modules/scripts/vdi-setup/README.md index c7389e8305..bb58f49995 100644 --- a/community/modules/scripts/vdi-setup/README.md +++ b/community/modules/scripts/vdi-setup/README.md @@ -1,38 +1,31 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ ## Description Creates a containerised Guacamole instance. Works with the Debian and Ubuntu images shown in blueprint example below. -### Service Startup and Health Checking - -The VDI module includes basic health checks to ensure services start properly: - ### Secret Manager Integration The VDI module supports flexible Secret Manager integration: - **Default Behavior**: Secrets are stored in the deployment project -- **Cross-Project Secrets**: Use `secret_project` to specify a different GCP project if providing user secrets - **Automatic Password Generation**: If no `secret_name` is provided, random passwords are generated and stored - **Existing Secret Retrieval**: Provide `secret_name` to use existing secrets from Secret Manager - -- **Database Health Check**: Waits for PostgreSQL to be ready on port 5432 (60 second timeout) -- **Guacamole Web Interface**: Checks HTTP endpoint availability -- **API Verification**: Verifies Guacamole admin login functionality -- **VNC Service Restart**: Systemd services configured with automatic restart on failure +- **Cross-Project Secrets**: Use `secret_project` to specify a different GCP project if providing secrets ## Outputs @@ -45,13 +38,13 @@ The module provides the following outputs: - `username`: The VDI user's username - `port`: The VNC port assigned to the user - `secret_name`: The name of the Secret Manager secret containing the user's password - - `secret_project`: The GCP project where the user's secret is stored (defaults to deployment project) + - `secret_project`: The GCP project where the user's secret is stored (defaults to deployment project) - `vdi_instance_ip`: The IP address of the VDI instance (null until VM is created) - `vdi_instance_name`: The name of the VDI instance (null until VM is created) ## Basic Example (Guacamole) -``` +```yaml blueprint_name: vdi-test vars: @@ -110,7 +103,7 @@ deployment_groups: source: modules/compute/vm-instance settings: instance_image: - # Several spported image families: + # Several supported image families: family: hpc-rocky-linux-8 project: cloud-hpc-image-public #family: debian-11 @@ -143,7 +136,7 @@ After deployment, you can access the VDI in several ways: 1. **Guacamole Web Interface**: - Access web interface: - http://$VM_PUBLIC_IP:8080/guacamole/#/ - - Note: It is not advisable to serve Guacamole directly from a public IP in production environments. Consider serving your VDI from behind a reverse proxy, load balancer, or tunnel to it directly over IAP (see below). + - Note: It is not advisable to serve Guacamole directly from a public IP in production environments. You should consider placing the VDI behind a reverse proxy, load balancer, or tunnel to it directly over IAP (see below). - Admin credentials: - Username: `guacadmin` @@ -155,11 +148,13 @@ After deployment, you can access the VDI in several ways: - Or use the custom secret name specified in the `vdi_users` configuration 3. **IAP Tunnel** (for development/testing): - ``` + + ```bash gcloud compute start-iap-tunnel vdi-test-0 8080 \ --local-host-port=localhost:8080 \ --zone=us-central1-a ``` + - Guacamole will then be accessible from http://localhost:8080/guacamole/ diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/debian.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/debian.yaml new file mode 100644 index 0000000000..b2999e86ec --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/debian.yaml @@ -0,0 +1,41 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Set Debian specific variables + ansible.builtin.set_fact: + gui_packages: ['xfce4', 'xserver-xorg-core', 'dbus-x11'] + vnc_pkg: >- + {%- if vnc_flavor|lower == 'tigervnc' -%} + tigervnc-standalone-server + {%- elif vnc_flavor|lower == 'tightvnc' -%} + tightvnc-standalone-server + {%- else -%} + tightvnc-standalone-server + {%- endif %} + xorg_dev_packages: ['pkg-config', 'xserver-xorg-dev', 'libx11-dev', 'libxext-dev', 'libglvnd-dev', 'build-essential'] + +- name: Install desktop prerequisites & VNC package (Debian) + ansible.builtin.package: + name: "{{ gui_packages + [vnc_pkg] + xorg_dev_packages }}" + state: present + retries: 10 + delay: 3 + +- name: Install kernel headers for running kernel (Debian) + ansible.builtin.package: + name: "linux-headers-{{ ansible_kernel }}" + state: present + retries: 10 + delay: 3 diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml index 9c67c60f56..f148682825 100644 --- a/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml @@ -49,26 +49,21 @@ {%- endif %} -# Block for NVIDIA vWS driver setup -- block: - - name: Install NVIDIA Vulkan ICD loader - ansible.builtin.package: - name: vulkan-loader - state: present - retries: 10 - delay: 3 - when: ansible_facts.os_family == 'RedHat' - - name: Download NVIDIA vWS driver - ansible.builtin.uri: - url: "{{ driver_url }}" - method: GET - dest: /tmp/nvidia-driver.run - mode: '0755' - status_code: - - 200 - - 304 - - name: Install NVIDIA vWS driver - ansible.builtin.command: - cmd: /tmp/nvidia-driver.run --dkms --silent - creates: /usr/bin/nvidia-smi - when: is_nvidia_vws_supported +# Include distribution-specific GPU driver setup +- name: Include Ubuntu GPU driver setup + ansible.builtin.include_tasks: gpu_driver_setup/ubuntu.yaml + when: + - ansible_facts.distribution == 'Ubuntu' + - is_nvidia_vws_supported + +- name: Include Rocky Linux GPU driver setup + ansible.builtin.include_tasks: gpu_driver_setup/rocky.yaml + when: + - ansible_facts.distribution == 'Rocky' + - is_nvidia_vws_supported + +- name: Include Debian GPU driver setup + ansible.builtin.include_tasks: gpu_driver_setup/debian.yaml + when: + - ansible_facts.distribution == 'Debian' + - is_nvidia_vws_supported diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/debian.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/debian.yaml new file mode 100644 index 0000000000..7bd4f3699c --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/debian.yaml @@ -0,0 +1,30 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# Block for NVIDIA vWS driver setup (Debian) +- block: + - name: Download NVIDIA vWS driver + ansible.builtin.uri: + url: "{{ driver_url }}" + method: GET + dest: /tmp/nvidia-driver.run + mode: '0755' + status_code: + - 200 + - 304 + - name: Install NVIDIA vWS driver (Debian) + ansible.builtin.command: + cmd: /tmp/nvidia-driver.run --dkms --silent + creates: /usr/bin/nvidia-smi diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/rocky.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/rocky.yaml new file mode 100644 index 0000000000..260ca90d19 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/rocky.yaml @@ -0,0 +1,36 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# Block for NVIDIA vWS driver setup (Rocky Linux) +- block: + - name: Install NVIDIA Vulkan ICD loader (Rocky Linux) + ansible.builtin.package: + name: vulkan-loader + state: present + retries: 10 + delay: 3 + - name: Download NVIDIA vWS driver + ansible.builtin.uri: + url: "{{ driver_url }}" + method: GET + dest: /tmp/nvidia-driver.run + mode: '0755' + status_code: + - 200 + - 304 + - name: Install NVIDIA vWS driver (Rocky Linux) + ansible.builtin.command: + cmd: /tmp/nvidia-driver.run --dkms --silent + creates: /usr/bin/nvidia-smi diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/ubuntu.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/ubuntu.yaml new file mode 100644 index 0000000000..8cef6585a0 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/ubuntu.yaml @@ -0,0 +1,73 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# Block for NVIDIA vWS driver setup (Ubuntu) +- block: + - name: Download NVIDIA vWS driver + ansible.builtin.uri: + url: "{{ driver_url }}" + method: GET + dest: /tmp/nvidia-driver.run + mode: '0755' + status_code: + - 200 + - 304 + + - name: Verify GCC version before driver installation + ansible.builtin.shell: gcc --version | head -1 + register: gcc_version_check + changed_when: false + + - name: Debug GCC version before driver installation + ansible.builtin.debug: + msg: "GCC version before driver installation: {{ gcc_version_check.stdout }}" + + - name: Install NVIDIA vWS driver (Ubuntu) + ansible.builtin.command: + cmd: /tmp/nvidia-driver.run --dkms --silent + creates: /usr/bin/nvidia-smi + register: nvidia_install_result + failed_when: false + + - name: Check if NVIDIA driver installation succeeded + ansible.builtin.stat: + path: /usr/bin/nvidia-smi + register: nvidia_smi_check + + - name: Debug NVIDIA installation result + ansible.builtin.debug: + msg: + - "NVIDIA installer exit code: {{ nvidia_install_result.rc }}" + - "nvidia-smi exists: {{ nvidia_smi_check.stat.exists }}" + - "Installation stdout: {{ nvidia_install_result.stdout }}" + - "Installation stderr: {{ nvidia_install_result.stderr }}" + + - name: Fail if NVIDIA driver installation failed + ansible.builtin.fail: + msg: "NVIDIA driver installation failed. Check /var/log/nvidia-installer.log for details." + when: + - nvidia_install_result.rc != 0 + - not nvidia_smi_check.stat.exists + + - name: Verify NVIDIA driver installation + ansible.builtin.command: nvidia-smi + register: nvidia_smi_test + changed_when: false + when: nvidia_smi_check.stat.exists + + - name: Debug NVIDIA driver verification + ansible.builtin.debug: + msg: "NVIDIA driver verification: {{ nvidia_smi_test.stdout }}" + when: nvidia_smi_check.stat.exists diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml index 0fec11a329..209155873b 100644 --- a/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml @@ -23,68 +23,20 @@ register: nvidia_gpu_present failed_when: false -- name: Set package manager and desktop-packages - ansible.builtin.set_fact: - # pkg_mgr: "{{ 'yum' if ansible_facts.os_family == 'RedHat' else 'apt' }}" - gui_packages: >- - {{ - ['@Xfce','@base-x'] - if ansible_facts.os_family == 'RedHat' - else - ( - ['xfce4', 'xserver-xorg-core', 'dbus-x11'] - if ansible_facts.distribution == 'Debian' - else - ['xfce4', 'xserver-xorg-core'] - ) - }} - vnc_pkg: >- - {%- if vnc_flavor|lower == 'tigervnc' and ansible_facts.os_family == 'RedHat' -%} - tigervnc-server - {%- elif vnc_flavor|lower == 'tigervnc' -%} - tigervnc-standalone-server - {%- elif vnc_flavor|lower == 'tightvnc' and ansible_facts.os_family == 'RedHat' -%} - tightvnc-server - {%- else -%} - tightvnc-standalone-server - {%- endif %} - xorg_dev_packages: >- - {{ - ['pkgconfig', 'xorg-x11-server-devel', 'xorg-x11-util-macros', 'libglvnd-devel'] - if ansible_facts.os_family == 'RedHat' - else - ( - ['pkg-config', 'xserver-xorg-dev', 'libx11-dev', 'libxext-dev', 'libglvnd-dev', 'build-essential'] - if ansible_facts.distribution == 'Debian' - else - ['pkg-config', 'xserver-xorg-dev', 'libx11-dev', 'libxext-dev', 'libglvnd-dev'] - ) - }} +# Include distribution-specific tasks +- name: Include Ubuntu-specific tasks + ansible.builtin.include_tasks: ubuntu.yaml + when: ansible_facts.distribution == 'Ubuntu' -- name: Update repository cache & install EPEL (RedHat only) - ansible.builtin.package: - name: epel-release - state: present - update_cache: yes - when: ansible_facts.os_family == 'RedHat' - retries: 10 - delay: 3 +- name: Include Rocky Linux specific tasks + ansible.builtin.include_tasks: rocky.yaml + when: ansible_facts.distribution == 'Rocky' -- name: Install desktop prerequisites & VNC package - ansible.builtin.package: - name: "{{ gui_packages + [vnc_pkg] + xorg_dev_packages }}" - state: present - retries: 10 - delay: 3 - -- name: Install kernel headers for running kernel (Debian only) - ansible.builtin.package: - name: "linux-headers-{{ ansible_kernel }}" - state: present +- name: Include Debian-specific tasks + ansible.builtin.include_tasks: debian.yaml when: ansible_facts.distribution == 'Debian' - retries: 10 - delay: 3 +# Common tasks for all distributions - name: Ensure requests package is installed pip: name: requests diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/rocky.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/rocky.yaml new file mode 100644 index 0000000000..132e53fa89 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/rocky.yaml @@ -0,0 +1,42 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Set Rocky Linux specific variables + ansible.builtin.set_fact: + gui_packages: ['@Xfce','@base-x'] + vnc_pkg: >- + {%- if vnc_flavor|lower == 'tigervnc' -%} + tigervnc-server + {%- elif vnc_flavor|lower == 'tightvnc' -%} + tightvnc-server + {%- else -%} + tightvnc-standalone-server + {%- endif %} + xorg_dev_packages: ['pkgconfig', 'xorg-x11-server-devel', 'xorg-x11-util-macros', 'libglvnd-devel'] + +- name: Update repository cache & install EPEL (Rocky Linux only) + ansible.builtin.package: + name: epel-release + state: present + update_cache: yes + retries: 10 + delay: 3 + +- name: Install desktop prerequisites & VNC package (Rocky Linux) + ansible.builtin.package: + name: "{{ gui_packages + [vnc_pkg] + xorg_dev_packages }}" + state: present + retries: 10 + delay: 3 diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml new file mode 100644 index 0000000000..e467a0cca1 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml @@ -0,0 +1,138 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Set Ubuntu specific variables + ansible.builtin.set_fact: + gui_packages: ['xfce4', 'xserver-xorg-core'] + vnc_pkg: >- + {%- if vnc_flavor|lower == 'tigervnc' -%} + tigervnc-standalone-server + {%- elif vnc_flavor|lower == 'tightvnc' -%} + tightvnc-standalone-server + {%- else -%} + tightvnc-standalone-server + {%- endif %} + xorg_dev_packages: ['pkg-config', 'xserver-xorg-dev', 'libx11-dev', 'libxext-dev', 'libglvnd-dev'] + +- name: Install desktop prerequisites & VNC package (Ubuntu) + ansible.builtin.package: + name: "{{ gui_packages + [vnc_pkg] + xorg_dev_packages }}" + state: present + retries: 10 + delay: 3 + +- name: Install kernel headers for running kernel (Ubuntu) + ansible.builtin.package: + name: "linux-headers-{{ ansible_kernel }}" + state: present + retries: 10 + delay: 3 + +# GCC version detection and installation for NVIDIA driver compatibility +- name: Get kernel build GCC version + ansible.builtin.shell: cat /proc/version | grep -o 'gcc-[0-9]*' | head -1 + register: kernel_gcc_version + changed_when: false + +- name: Extract GCC major version from kernel + ansible.builtin.set_fact: + required_gcc_version: "{{ kernel_gcc_version.stdout.split('-')[1] if kernel_gcc_version.stdout else '11' }}" + +- name: Debug kernel GCC version extraction + ansible.builtin.debug: + msg: + - "Raw kernel GCC version: {{ kernel_gcc_version.stdout }}" + - "Extracted required GCC version: {{ required_gcc_version }}" + +- name: Get current GCC version + ansible.builtin.shell: gcc --version | head -1 | grep -o '[0-9]*\.[0-9]*' | head -1 + register: current_gcc_version + changed_when: false + +- name: Extract current GCC major version + ansible.builtin.set_fact: + current_gcc_major: "{{ current_gcc_version.stdout.split('.')[0] if current_gcc_version.stdout else '11' }}" + +- name: Debug current GCC version extraction + ansible.builtin.debug: + msg: + - "Raw current GCC version: {{ current_gcc_version.stdout }}" + - "Extracted current GCC major: {{ current_gcc_major }}" + +- name: Debug GCC version information + ansible.builtin.debug: + msg: + - "Kernel was built with GCC {{ required_gcc_version }}" + - "Current system GCC version: {{ current_gcc_major }}" + - "GCC update needed: {{ required_gcc_version != current_gcc_major }}" + +- name: Install required GCC version if different from current + block: + - name: Install GCC {{ required_gcc_version }} + ansible.builtin.package: + name: "gcc-{{ required_gcc_version }}" + state: present + retries: 10 + delay: 3 + + - name: Install G++ {{ required_gcc_version }} + ansible.builtin.package: + name: "g++-{{ required_gcc_version }}" + state: present + retries: 10 + delay: 3 + + - name: Set GCC {{ required_gcc_version }} as default + ansible.builtin.alternatives: + name: gcc + path: "/usr/bin/gcc-{{ required_gcc_version }}" + priority: "{{ required_gcc_version }}" + link: /usr/bin/gcc + + - name: Set G++ {{ required_gcc_version }} as default + ansible.builtin.alternatives: + name: g++ + path: "/usr/bin/g++-{{ required_gcc_version }}" + priority: "{{ required_gcc_version }}" + link: /usr/bin/g++ + + - name: Verify GCC version after update + ansible.builtin.shell: gcc --version | head -1 + register: updated_gcc_version + changed_when: false + + - name: Debug updated GCC version + ansible.builtin.debug: + msg: "Updated GCC version: {{ updated_gcc_version.stdout }}" + + - name: Verify G++ version after update + ansible.builtin.shell: g++ --version | head -1 + register: updated_gpp_version + changed_when: false + + - name: Debug updated G++ version + ansible.builtin.debug: + msg: "Updated G++ version: {{ updated_gpp_version.stdout }}" + + - name: Verify alternatives are set correctly + ansible.builtin.shell: update-alternatives --display gcc + register: gcc_alternatives_check + changed_when: false + + - name: Debug GCC alternatives status + ansible.builtin.debug: + msg: "GCC alternatives status: {{ gcc_alternatives_check.stdout }}" + + when: required_gcc_version != current_gcc_major diff --git a/community/modules/scripts/vdi-setup/versions.tf b/community/modules/scripts/vdi-setup/versions.tf new file mode 100644 index 0000000000..6024b738a7 --- /dev/null +++ b/community/modules/scripts/vdi-setup/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.0" + required_providers { + archive = { + source = "hashicorp/archive" + version = "~> 2.0" + } + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } +} From b8d13956490c678718b858928fd5a5d426b02c7e Mon Sep 17 00:00:00 2001 From: Scott Gordon Date: Thu, 17 Jul 2025 15:14:07 +0100 Subject: [PATCH 05/21] formatting --- community/modules/scripts/vdi-setup/README.md | 26 +++++-------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/community/modules/scripts/vdi-setup/README.md b/community/modules/scripts/vdi-setup/README.md index bb58f49995..37b1e59b36 100644 --- a/community/modules/scripts/vdi-setup/README.md +++ b/community/modules/scripts/vdi-setup/README.md @@ -1,19 +1,3 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - ## Description Creates a containerised Guacamole instance. Works with the Debian and Ubuntu images shown in blueprint example below. @@ -160,14 +144,18 @@ After deployment, you can access the VDI in several ways: ## Requirements -No requirements. +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [archive](#requirement\_archive) | ~> 2.0 | +| [google](#requirement\_google) | >= 3.83 | ## Providers | Name | Version | |------|---------| -| [archive](#provider\_archive) | n/a | -| [google](#provider\_google) | n/a | +| [archive](#provider\_archive) | ~> 2.0 | +| [google](#provider\_google) | >= 3.83 | | [terraform](#provider\_terraform) | n/a | ## Modules From 6bd25397cfc796a369300703fd3c32aa20dd398a Mon Sep 17 00:00:00 2001 From: Scott Gordon Date: Thu, 17 Jul 2025 15:15:53 +0100 Subject: [PATCH 06/21] docs --- community/modules/scripts/vdi-setup/README.md | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/community/modules/scripts/vdi-setup/README.md b/community/modules/scripts/vdi-setup/README.md index 37b1e59b36..b0bec2f2b2 100644 --- a/community/modules/scripts/vdi-setup/README.md +++ b/community/modules/scripts/vdi-setup/README.md @@ -11,21 +11,6 @@ The VDI module supports flexible Secret Manager integration: - **Existing Secret Retrieval**: Provide `secret_name` to use existing secrets from Secret Manager - **Cross-Project Secrets**: Use `secret_project` to specify a different GCP project if providing secrets -## Outputs - -The module provides the following outputs: - -- `guacamole_url`: The URL to access the Guacamole web interface (null until VM is created) -- `guacamole_admin_username`: The admin username for Guacamole (default: "guacadmin") -- `guacamole_admin_password_secret`: The name of the Secret Manager secret containing the Guacamole admin password - - `vdi_user_credentials`: A map of VDI user credentials stored in Secret Manager, including: - - `username`: The VDI user's username - - `port`: The VNC port assigned to the user - - `secret_name`: The name of the Secret Manager secret containing the user's password - - `secret_project`: The GCP project where the user's secret is stored (defaults to deployment project) -- `vdi_instance_ip`: The IP address of the VDI instance (null until VM is created) -- `vdi_instance_name`: The name of the VDI instance (null until VM is created) - ## Basic Example (Guacamole) ```yaml From 573bedcc0ebbbd7b3fca0f4927f3cc08d720a2bd Mon Sep 17 00:00:00 2001 From: Scott Gordon Date: Thu, 17 Jul 2025 15:23:17 +0100 Subject: [PATCH 07/21] vdi module --- @ | 66 --------------------------------------------------------------- 1 file changed, 66 deletions(-) delete mode 100644 @ diff --git a/@ b/@ deleted file mode 100644 index 5e2392bfcd..0000000000 --- a/@ +++ /dev/null @@ -1,66 +0,0 @@ - -# Default compute SA will require IAM role: Secret Manager Secret Accessor - -blueprint_name: vdi-test-scott - -vars: - deployment_name: vdi-test-scott - project_id: hpc-discovery-external - region: us-central1 - zone: us-central1-a - -deployment_groups: -- group: primary - modules: - - id: network1 - source: modules/network/vpc - settings: - extra_iap_ports: [8080] - firewall_rules: - - name: allow-guacamole-8080-ext - description: Allow external ingress to Guacamole on TCP port 8080 - direction: INGRESS - ranges: ["0.0.0.0/0"] # or rev proxy address? - allow: - - protocol: tcp - ports: ["8080"] - - - id: enable-apis - source: community/modules/project/service-enablement - settings: - gcp_service_list: - - secretmanager.googleapis.com - - storage.googleapis.com - - compute.googleapis.com - - - id: vdi-setup - source: community/modules/scripts/vdi-setup - settings: - vnc_flavor: tigervnc - vdi_tool: guacamole - user_provision: local_users - vdi_user_group: vdiusers - vdi_resolution: 1920x1080 - vdi_users: - - username: alice - port: 5901 - - username: bob - port: 5902 - secret_name: a-password-for-bob - - - id: machine - source: modules/compute/vm-instance - settings: - instance_image: - #family: debian-11 - #project: debian-cloud - #family: ubuntu-2204-lts - #project: ubuntu-os-cloud - family: hpc-rocky-linux-8 - project: cloud-hpc-image-public - machine_type: g2-standard-8 - disable_public_ips: false - tags: ["guacamole"] - use: - - network1 - - vdi-setup From ee6a348fec19292a4c90aa601158cd310603c570 Mon Sep 17 00:00:00 2001 From: Scott Gordon Date: Thu, 17 Jul 2025 15:33:03 +0100 Subject: [PATCH 08/21] vdi module --- community/modules/scripts/vdi-setup/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/modules/scripts/vdi-setup/README.md b/community/modules/scripts/vdi-setup/README.md index b0bec2f2b2..1bcd0c46fe 100644 --- a/community/modules/scripts/vdi-setup/README.md +++ b/community/modules/scripts/vdi-setup/README.md @@ -1,6 +1,6 @@ ## Description -Creates a containerised Guacamole instance. Works with the Debian and Ubuntu images shown in blueprint example below. +Creates a containerised Guacamole instance. Works with the Rocky, Debian and Ubuntu images shown in blueprint example below. ### Secret Manager Integration From df9f901a40c5ece50a3c5cfd013031e71a332774 Mon Sep 17 00:00:00 2001 From: Scott Gordon Date: Thu, 17 Jul 2025 15:33:56 +0100 Subject: [PATCH 09/21] vdi module --- community/modules/scripts/vdi-setup/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/modules/scripts/vdi-setup/README.md b/community/modules/scripts/vdi-setup/README.md index 1bcd0c46fe..94330f6362 100644 --- a/community/modules/scripts/vdi-setup/README.md +++ b/community/modules/scripts/vdi-setup/README.md @@ -33,7 +33,7 @@ deployment_groups: - name: allow-guacamole-8080-ext description: Allow external ingress to Guacamole on TCP port 8080 direction: INGRESS - ranges: ["0.0.0.0/0"] # or rev proxy address? + ranges: ["0.0.0.0/0"] allow: - protocol: tcp ports: ["8080"] From e3e815e7ad64f7989b6e3a7cfa947478c47fec5e Mon Sep 17 00:00:00 2001 From: Scott Gordon Date: Thu, 17 Jul 2025 15:41:09 +0100 Subject: [PATCH 10/21] vdi module --- community/modules/scripts/vdi-setup/README.md | 5 ----- community/modules/scripts/vdi-setup/outputs.tf | 15 --------------- community/modules/scripts/vdi-setup/variables.tf | 12 ------------ 3 files changed, 32 deletions(-) diff --git a/community/modules/scripts/vdi-setup/README.md b/community/modules/scripts/vdi-setup/README.md index 94330f6362..1a3e4cf7e6 100644 --- a/community/modules/scripts/vdi-setup/README.md +++ b/community/modules/scripts/vdi-setup/README.md @@ -166,8 +166,6 @@ After deployment, you can access the VDI in several ways: | [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | | [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | | [user\_provision](#input\_user\_provision) | User type to create (local\_users supported. os-login to do. | `string` | `"local_users"` | no | -| [vdi\_instance\_ip](#input\_vdi\_instance\_ip) | The IP address of the VDI instance | `string` | `null` | no | -| [vdi\_instance\_name](#input\_vdi\_instance\_name) | The name of the VDI instance | `string` | `null` | no | | [vdi\_resolution](#input\_vdi\_resolution) | Desktop resolution for VNC sessions (e.g. 1920x1080). | `string` | `"1920x1080"` | no | | [vdi\_tool](#input\_vdi\_tool) | VDI tool to deploy (guacamole currently supported). | `string` | `"guacamole"` | no | | [vdi\_user\_group](#input\_vdi\_user\_group) | Unix group to create/use for VDI users. | `string` | `"vdiusers"` | no | @@ -183,10 +181,7 @@ After deployment, you can access the VDI in several ways: |------|-------------| | [guacamole\_admin\_password\_secret](#output\_guacamole\_admin\_password\_secret) | The name of the Secret Manager secret containing the Guacamole admin password | | [guacamole\_admin\_username](#output\_guacamole\_admin\_username) | The admin username for Guacamole | -| [guacamole\_url](#output\_guacamole\_url) | The URL to access the Guacamole web interface | | [startup\_script](#output\_startup\_script) | Combined startup script that installs VDI (VNC, Guacamole, users). | -| [vdi\_instance\_ip](#output\_vdi\_instance\_ip) | The IP address of the VDI instance | -| [vdi\_instance\_name](#output\_vdi\_instance\_name) | The name of the VDI instance | | [vdi\_runner](#output\_vdi\_runner) | Shell runner wrapping Ansible playbook + roles (for custom-image or direct use). | | [vdi\_user\_credentials](#output\_vdi\_user\_credentials) | Map of VDI user credentials stored in Secret Manager | diff --git a/community/modules/scripts/vdi-setup/outputs.tf b/community/modules/scripts/vdi-setup/outputs.tf index f3a9787644..4e8fa9f21e 100644 --- a/community/modules/scripts/vdi-setup/outputs.tf +++ b/community/modules/scripts/vdi-setup/outputs.tf @@ -22,11 +22,6 @@ output "vdi_runner" { value = local.combined_runner } -output "guacamole_url" { - description = "The URL to access the Guacamole web interface" - value = var.vdi_instance_ip != null ? "http://${var.vdi_instance_ip}:${var.vdi_webapp_port}/guacamole/" : null -} - output "guacamole_admin_username" { description = "The admin username for Guacamole" value = "guacadmin" @@ -47,13 +42,3 @@ output "vdi_user_credentials" { } } } - -output "vdi_instance_ip" { - description = "The IP address of the VDI instance" - value = var.vdi_instance_ip -} - -output "vdi_instance_name" { - description = "The name of the VDI instance" - value = var.vdi_instance_name -} diff --git a/community/modules/scripts/vdi-setup/variables.tf b/community/modules/scripts/vdi-setup/variables.tf index 9b1ce654ab..38cf7bd42a 100644 --- a/community/modules/scripts/vdi-setup/variables.tf +++ b/community/modules/scripts/vdi-setup/variables.tf @@ -89,15 +89,3 @@ variable "vnc_port_max" { default = 5999 description = "Maximum valid VNC port." } - -variable "vdi_instance_ip" { - description = "The IP address of the VDI instance" - type = string - default = null -} - -variable "vdi_instance_name" { - description = "The name of the VDI instance" - type = string - default = null -} From acc23c383364380c66546713b95e1c8f878ad36a Mon Sep 17 00:00:00 2001 From: Scott Gordon Date: Fri, 18 Jul 2025 10:37:32 +0100 Subject: [PATCH 11/21] pre-commit tweaks --- .../roles/base_os/tasks/gpu_driver_setup.yaml | 14 +- .../tasks/gpu_driver_setup/debian.yaml | 26 +- .../base_os/tasks/gpu_driver_setup/rocky.yaml | 38 +- .../tasks/gpu_driver_setup/ubuntu.yaml | 96 +- .../vdi-setup/roles/base_os/tasks/main.yaml | 6 +- .../vdi-setup/roles/base_os/tasks/rocky.yaml | 2 +- .../vdi-setup/roles/base_os/tasks/ubuntu.yaml | 126 +- .../tasks/user_secret_tasks.yaml | 8 +- .../user_provision/tasks/local_users.yaml | 60 +- .../roles/user_provision/tasks/main.yaml | 2 +- .../roles/vdi_tool/tasks/guacamole.yaml | 38 +- .../vdi-setup/roles/vnc/tasks/main.yaml | 1 - .../startup-script/files/install_docker.yml | 46 +- .../artifacts/DO_NOT_MODIFY_THIS_DIRECTORY | 1 + .../.ghpc/artifacts/expanded_blueprint.yaml | 112 + vdi-test-scott/.gitignore | 48 + vdi-test-scott/instructions.txt | 23 + vdi-test-scott/primary/main.tf | 80 + .../embedded/community/modules/README.md | 7 + .../compute/htcondor-execute-point/README.md | 271 + .../htcondor-execute-point/compute_image.tf | 30 + .../files/htcondor_configure.yml | 74 + .../files/htcondor_configure_autoscaler.yml | 98 + .../compute/htcondor-execute-point/main.tf | 218 + .../htcondor-execute-point/metadata.yaml | 20 + .../compute/htcondor-execute-point/outputs.tf | 25 + .../templates/condor_config.tftpl | 31 + .../download-condor-config.ps1.tftpl | 34 + .../htcondor-execute-point/variables.tf | 265 + .../htcondor-execute-point/versions.tf | 34 + .../community/modules/compute/mig/README.md | 45 + .../community/modules/compute/mig/main.tf | 85 + .../modules/compute/mig/metadata.yaml | 21 + .../community/modules/compute/mig/outputs.tf | 18 + .../modules/compute/mig/variables.tf | 86 + .../community/modules/compute/mig/versions.tf | 27 + .../modules/compute/notebook/README.md | 112 + .../modules/compute/notebook/main.tf | 96 + .../modules/compute/notebook/metadata.yaml | 20 + .../modules/compute/notebook/variables.tf | 111 + .../modules/compute/notebook/versions.tf | 29 + .../README.md | 139 + .../main.tf | 121 + .../metadata.yaml | 20 + .../outputs.tf | 36 + .../source_image_logic.tf | 77 + .../variables.tf | 402 + .../versions.tf | 29 + .../README.md | 85 + .../schedmd-slurm-gcp-v6-nodeset-tpu/main.tf | 59 + .../metadata.yaml | 21 + .../outputs.tf | 39 + .../variables.tf | 171 + .../versions.tf | 23 + .../schedmd-slurm-gcp-v6-nodeset/README.md | 226 + .../schedmd-slurm-gcp-v6-nodeset/main.tf | 224 + .../metadata.yaml | 21 + .../schedmd-slurm-gcp-v6-nodeset/outputs.tf | 102 + .../source_image_logic.tf | 77 + .../schedmd-slurm-gcp-v6-nodeset/variables.tf | 635 + .../schedmd-slurm-gcp-v6-nodeset/versions.tf | 29 + .../schedmd-slurm-gcp-v6-partition/README.md | 105 + .../schedmd-slurm-gcp-v6-partition/main.tf | 42 + .../metadata.yaml | 20 + .../schedmd-slurm-gcp-v6-partition/outputs.tf | 59 + .../variables.tf | 310 + .../versions.tf | 23 + .../container/artifact-registry/README.md | 157 + .../container/artifact-registry/main.tf | 268 + .../container/artifact-registry/metadata.yaml | 21 + .../container/artifact-registry/outputs.tf | 18 + .../container/artifact-registry/validation.tf | 49 + .../container/artifact-registry/variables.tf | 122 + .../container/artifact-registry/versions.tf | 27 + .../database/bigquery-dataset/README.md | 76 + .../modules/database/bigquery-dataset/main.tf | 32 + .../database/bigquery-dataset/metadata.yaml | 19 + .../database/bigquery-dataset/outputs.tf | 20 + .../database/bigquery-dataset/variables.tf | 36 + .../database/bigquery-dataset/versions.tf | 29 + .../modules/database/bigquery-table/README.md | 87 + .../modules/database/bigquery-table/main.tf | 37 + .../database/bigquery-table/metadata.yaml | 19 + .../database/bigquery-table/outputs.tf | 28 + .../database/bigquery-table/variables.tf | 46 + .../database/bigquery-table/versions.tf | 29 + .../slurm-cloudsql-federation/README.md | 107 + .../slurm-cloudsql-federation/main.tf | 165 + .../slurm-cloudsql-federation/metadata.yaml | 21 + .../slurm-cloudsql-federation/outputs.tf | 27 + .../slurm-cloudsql-federation/variables.tf | 173 + .../slurm-cloudsql-federation/versions.tf | 36 + .../file-system/DDN-EXAScaler/README.md | 158 + .../modules/file-system/DDN-EXAScaler/main.tf | 72 + .../file-system/DDN-EXAScaler/metadata.yaml | 22 + .../file-system/DDN-EXAScaler/outputs.tf | 90 + .../file-system/DDN-EXAScaler/variables.tf | 502 + .../file-system/DDN-EXAScaler/versions.tf | 24 + .../modules/file-system/Intel-DAOS/README.md | 1 + .../cloud-storage-bucket/README.md | 170 + .../file-system/cloud-storage-bucket/main.tf | 126 + .../cloud-storage-bucket/metadata.yaml | 18 + .../cloud-storage-bucket/outputs.tf | 69 + .../scripts/install-gcs-fuse.sh | 44 + .../cloud-storage-bucket/scripts/mount.sh | 58 + .../cloud-storage-bucket/variables.tf | 254 + .../cloud-storage-bucket/versions.tf | 39 + .../modules/file-system/nfs-server/README.md | 152 + .../modules/file-system/nfs-server/main.tf | 131 + .../file-system/nfs-server/metadata.yaml | 19 + .../modules/file-system/nfs-server/outputs.tf | 53 + .../nfs-server/scripts/install-nfs-client.sh | 37 + .../scripts/install-nfs-server.sh.tpl | 35 + .../file-system/nfs-server/scripts/mount.sh | 58 + .../file-system/nfs-server/scripts/mount.yaml | 39 + .../file-system/nfs-server/variables.tf | 194 + .../file-system/nfs-server/versions.tf | 37 + .../FSI_MonteCarlo.ipynb | 125 + .../files/fsi-montecarlo-on-batch/README.md | 97 + .../fsi-montecarlo-on-batch/iteration.sh | 23 + .../files/fsi-montecarlo-on-batch/main.tf | 102 + .../fsi-montecarlo-on-batch/mc_run.tpl.py | 157 + .../fsi-montecarlo-on-batch/mc_run.tpl.yaml | 36 + .../fsi-montecarlo-on-batch/mc_run_reqs.txt | 9 + .../fsi-montecarlo-on-batch/metadata.yaml | 18 + .../fsi-montecarlo-on-batch/variables.tf | 51 + .../files/fsi-montecarlo-on-batch/versions.tf | 43 + .../internal/slurm-gcp/instance/README.md | 100 + .../internal/slurm-gcp/instance/main.tf | 126 + .../internal/slurm-gcp/instance/outputs.tf | 41 + .../internal/slurm-gcp/instance/variables.tf | 119 + .../internal/slurm-gcp/instance/versions.tf | 31 + .../slurm-gcp/instance_template/README.md | 86 + .../files/startup_sh_unlinted | 161 + .../slurm-gcp/instance_template/main.tf | 164 + .../slurm-gcp/instance_template/outputs.tf | 43 + .../slurm-gcp/instance_template/variables.tf | 431 + .../slurm-gcp/instance_template/versions.tf | 25 + .../internal_instance_template/README.md | 87 + .../internal_instance_template/main.tf | 227 + .../internal_instance_template/outputs.tf | 33 + .../internal_instance_template/variables.tf | 398 + .../internal_instance_template/versions.tf | 30 + .../internal/slurm-gcp/login/README.md | 52 + .../modules/internal/slurm-gcp/login/main.tf | 110 + .../internal/slurm-gcp/login/outputs.tf | 25 + .../internal/slurm-gcp/login/variables.tf | 188 + .../internal/slurm-gcp/login/versions.tf | 29 + .../internal/slurm-gcp/nodeset_tpu/README.md | 95 + .../internal/slurm-gcp/nodeset_tpu/main.tf | 121 + .../internal/slurm-gcp/nodeset_tpu/outputs.tf | 30 + .../slurm-gcp/nodeset_tpu/variables.tf | 158 + .../slurm-gcp/nodeset_tpu/versions.tf | 30 + .../dependencies-installer/README.md | 61 + .../helm_install/README.md | 64 + .../helm_install/main.tf | 75 + .../helm_install/metadata.yaml | 19 + .../helm_install/variables.tf | 212 + .../helm_install/versions.tf | 24 + .../kubernetes_manifest/README.md | 40 + .../kubernetes_manifest/main.tf | 104 + .../kubernetes_manifest/metadata.yaml | 19 + .../kubernetes_manifest/variables.tf | 69 + .../kubernetes_manifest/versions.tf | 24 + .../management/dependencies-installer/main.tf | 183 + .../dependencies-installer/metadata.yaml | 19 + .../dependencies-installer/providers.tf | 25 + .../dependencies-installer/variables.tf | 70 + .../dependencies-installer/versions.tf | 30 + .../network/private-service-access/README.md | 98 + .../network/private-service-access/main.tf | 42 + .../private-service-access/metadata.yaml | 20 + .../network/private-service-access/outputs.tf | 43 + .../private-service-access/variables.tf | 47 + .../private-service-access/versions.tf | 41 + .../modules/project/new-project/README.md | 128 + .../modules/project/service-account/README.md | 111 + .../modules/project/service-account/main.tf | 37 + .../project/service-account/metadata.yaml | 19 + .../project/service-account/outputs.tf | 36 + .../project/service-account/variables.tf | 113 + .../project/service-account/versions.tf | 22 + .../project/service-enablement/README.md | 70 + .../project/service-enablement/main.tf | 28 + .../project/service-enablement/metadata.yaml | 19 + .../project/service-enablement/variables.tf | 31 + .../project/service-enablement/versions.tf | 29 + .../modules/pubsub/bigquery-sub/README.md | 87 + .../modules/pubsub/bigquery-sub/main.tf | 57 + .../modules/pubsub/bigquery-sub/metadata.yaml | 19 + .../modules/pubsub/bigquery-sub/outputs.tf | 20 + .../modules/pubsub/bigquery-sub/variables.tf | 51 + .../modules/pubsub/bigquery-sub/versions.tf | 35 + .../community/modules/pubsub/topic/README.md | 82 + .../community/modules/pubsub/topic/main.tf | 48 + .../modules/pubsub/topic/metadata.yaml | 19 + .../community/modules/pubsub/topic/outputs.tf | 26 + .../modules/pubsub/topic/variables.tf | 74 + .../modules/pubsub/topic/versions.tf | 32 + .../chrome-remote-desktop/README.md | 113 + .../chrome-remote-desktop/main.tf | 111 + .../chrome-remote-desktop/metadata.yaml | 18 + .../chrome-remote-desktop/outputs.tf | 25 + .../scripts/configure-chrome-desktop.yml | 61 + .../scripts/configure-grid-drivers.yml | 163 + .../scripts/disable-sleep.yml | 39 + .../chrome-remote-desktop/variables.tf | 277 + .../chrome-remote-desktop/versions.tf | 19 + .../scheduler/htcondor-access-point/README.md | 187 + .../files/htcondor_configure.yml | 120 + .../scheduler/htcondor-access-point/main.tf | 338 + .../htcondor-access-point/metadata.yaml | 20 + .../htcondor-access-point/outputs.tf | 25 + .../templates/condor_config.tftpl | 70 + .../htcondor-access-point/variables.tf | 266 + .../htcondor-access-point/versions.tf | 37 + .../htcondor-central-manager/README.md | 159 + .../files/htcondor_configure.yml | 72 + .../htcondor-central-manager/main.tf | 226 + .../htcondor-central-manager/metadata.yaml | 20 + .../htcondor-central-manager/outputs.tf | 30 + .../templates/condor_config.tftpl | 31 + .../htcondor-central-manager/variables.tf | 192 + .../htcondor-central-manager/versions.tf | 33 + .../scheduler/htcondor-pool-secrets/README.md | 172 + .../files/htcondor_secrets.yml | 102 + .../scheduler/htcondor-pool-secrets/main.tf | 168 + .../htcondor-pool-secrets/metadata.yaml | 20 + .../htcondor-pool-secrets/outputs.tf | 50 + .../templates/fetch-idtoken.ps1.tftpl | 26 + .../htcondor-pool-secrets/variables.tf | 67 + .../htcondor-pool-secrets/versions.tf | 33 + .../htcondor-service-accounts/README.md | 128 + .../htcondor-service-accounts/main.tf | 51 + .../htcondor-service-accounts/metadata.yaml | 19 + .../htcondor-service-accounts/outputs.tf | 30 + .../htcondor-service-accounts/variables.tf | 56 + .../htcondor-service-accounts/versions.tf | 19 + .../scheduler/htcondor-setup/README.md | 118 + .../modules/scheduler/htcondor-setup/main.tf | 68 + .../scheduler/htcondor-setup/metadata.yaml | 21 + .../scheduler/htcondor-setup/outputs.tf | 27 + .../scheduler/htcondor-setup/variables.tf | 55 + .../scheduler/htcondor-setup/versions.tf | 19 + .../schedmd-slurm-gcp-v6-controller/README.md | 405 + .../controller.tf | 213 + .../etc/htc-slurm.conf.tpl | 65 + .../etc/htc-slurmdbd.conf.tpl | 34 + .../etc/long-prolog-slurm.conf.tpl | 71 + .../schedmd-slurm-gcp-v6-controller/login.tf | 50 + .../schedmd-slurm-gcp-v6-controller/main.tf | 35 + .../metadata.yaml | 21 + .../modules/cleanup_compute/README.md | 42 + .../modules/cleanup_compute/main.tf | 46 + .../scripts/cleanup_compute.sh | 92 + .../modules/cleanup_compute/variables.tf | 71 + .../modules/cleanup_compute/versions.tf | 27 + .../modules/cleanup_tpu/README.md | 79 + .../modules/cleanup_tpu/main.tf | 32 + .../cleanup_tpu/scripts/cleanup_tpu.sh | 63 + .../modules/cleanup_tpu/variables.tf | 60 + .../modules/cleanup_tpu/versions.tf | 27 + .../modules/slurm_files/README.md | 119 + .../modules/slurm_files/etc/cgroup.conf.tpl | 7 + .../modules/slurm_files/etc/slurm.conf.tpl | 67 + .../modules/slurm_files/etc/slurmdbd.conf.tpl | 31 + .../slurm_files/files/external_epilog.sh | 18 + .../slurm_files/files/external_prolog.sh | 18 + .../slurm_files/files/setup_external.sh | 115 + .../modules/slurm_files/main.tf | 331 + .../modules/slurm_files/outputs.tf | 45 + .../modules/slurm_files/scripts/conf.py | 641 + .../modules/slurm_files/scripts/file_cache.py | 80 + .../slurm_files/scripts/get_tpu_vmcount.py | 76 + .../slurm_files/scripts/job_submit.lua.tpl | 103 + .../modules/slurm_files/scripts/load_bq.py | 352 + .../slurm_files/scripts/local_pubsub.py | 196 + .../modules/slurm_files/scripts/mig_flex.py | 214 + .../slurm_files/scripts/requirements-dev.txt | 9 + .../slurm_files/scripts/requirements.txt | 18 + .../modules/slurm_files/scripts/resume.py | 664 + .../slurm_files/scripts/resume_wrapper.sh | 41 + .../modules/slurm_files/scripts/setup.py | 648 + .../scripts/setup_network_storage.py | 327 + .../modules/slurm_files/scripts/slurmsync.py | 661 + .../modules/slurm_files/scripts/sort_nodes.py | 171 + .../modules/slurm_files/scripts/suspend.py | 126 + .../slurm_files/scripts/suspend_wrapper.sh | 28 + .../slurm_files/scripts/tests/common.py | 115 + .../slurm_files/scripts/tests/test_conf.py | 192 + .../slurm_files/scripts/tests/test_resume.py | 175 + .../scripts/tests/test_topology.py | 215 + .../slurm_files/scripts/tests/test_util.py | 586 + .../slurm_files/scripts/tools/gpu-test | 129 + .../slurm_files/scripts/tools/task-epilog | 67 + .../slurm_files/scripts/tools/task-prolog | 70 + .../modules/slurm_files/scripts/tpu.py | 331 + .../modules/slurm_files/scripts/util.py | 2148 +++ .../slurm_files/scripts/watch_delete_vm_op.py | 124 + .../modules/slurm_files/variables.tf | 500 + .../modules/slurm_files/versions.tf | 37 + .../outputs.tf | 57 + .../partition.tf | 172 + .../slurm_files.tf | 191 + .../source_image_logic.tf | 77 + .../variables.tf | 809 + .../variables_controller_instance.tf | 382 + .../versions.tf | 33 + .../schedmd-slurm-gcp-v6-login/README.md | 134 + .../schedmd-slurm-gcp-v6-login/main.tf | 108 + .../schedmd-slurm-gcp-v6-login/metadata.yaml | 21 + .../schedmd-slurm-gcp-v6-login/outputs.tf | 18 + .../source_image_logic.tf | 77 + .../schedmd-slurm-gcp-v6-login/variables.tf | 419 + .../schedmd-slurm-gcp-v6-login/versions.tf | 29 + .../scripts/htcondor-install/README.md | 149 + .../htcondor-install/files/autoscaler.py | 417 + .../install-htcondor-autoscaler-deps.yml | 46 + .../files/install-htcondor.yaml | 94 + .../modules/scripts/htcondor-install/main.tf | 51 + .../scripts/htcondor-install/metadata.yaml | 18 + .../scripts/htcondor-install/outputs.tf | 30 + .../templates/install-htcondor.ps1.tftpl | 59 + .../scripts/htcondor-install/variables.tf | 51 + .../scripts/htcondor-install/versions.tf | 19 + .../modules/scripts/ramble-execute/README.md | 116 + .../modules/scripts/ramble-execute/main.tf | 71 + .../scripts/ramble-execute/metadata.yaml | 18 + .../modules/scripts/ramble-execute/outputs.tf | 53 + .../templates/ramble_execute.yml.tpl | 59 + .../scripts/ramble-execute/variables.tf | 114 + .../scripts/ramble-execute/versions.tf | 25 + .../modules/scripts/ramble-setup/README.md | 128 + .../modules/scripts/ramble-setup/main.tf | 113 + .../scripts/ramble-setup/metadata.yaml | 18 + .../modules/scripts/ramble-setup/outputs.tf | 61 + .../scripts/install_ramble_deps.yml | 50 + .../install_ramble_python_deps.yml.tftpl | 28 + .../templates/ramble_setup.yml.tftpl | 157 + .../modules/scripts/ramble-setup/variables.tf | 97 + .../modules/scripts/ramble-setup/versions.tf | 30 + .../modules/scripts/spack-execute/README.md | 141 + .../modules/scripts/spack-execute/main.tf | 70 + .../scripts/spack-execute/metadata.yaml | 18 + .../modules/scripts/spack-execute/outputs.tf | 45 + .../templates/execute_commands.yml.tpl | 59 + .../scripts/spack-execute/variables.tf | 103 + .../modules/scripts/spack-execute/versions.tf | 25 + .../modules/scripts/spack-setup/README.md | 382 + .../modules/scripts/spack-setup/main.tf | 120 + .../modules/scripts/spack-setup/metadata.yaml | 19 + .../modules/scripts/spack-setup/outputs.tf | 56 + .../scripts/install_spack_deps.yml | 50 + .../templates/spack_setup.yml.tftpl | 157 + .../modules/scripts/spack-setup/variables.tf | 106 + .../modules/scripts/spack-setup/versions.tf | 30 + .../modules/scripts/vdi-setup/README.md | 187 + .../modules/scripts/vdi-setup/main.tf | 134 + .../modules/scripts/vdi-setup/outputs.tf | 44 + .../modules/scripts/vdi-setup/roles.tar.gz | Bin 0 -> 8732 bytes .../vdi-setup/roles/base_os/tasks/debian.yaml | 41 + .../roles/base_os/tasks/gpu_driver_setup.yaml | 69 + .../tasks/gpu_driver_setup/debian.yaml | 30 + .../base_os/tasks/gpu_driver_setup/rocky.yaml | 36 + .../tasks/gpu_driver_setup/ubuntu.yaml | 73 + .../vdi-setup/roles/base_os/tasks/main.yaml | 56 + .../vdi-setup/roles/base_os/tasks/rocky.yaml | 42 + .../vdi-setup/roles/base_os/tasks/ubuntu.yaml | 138 + .../roles/secret_manager/defaults/main.yaml | 16 + .../roles/secret_manager/tasks/main.yaml | 102 + .../tasks/user_secret_tasks.yaml | 127 + .../user_provision/tasks/local_users.yaml | 51 + .../roles/user_provision/tasks/main.yaml | 40 + .../roles/user_provision/tasks/os_login.yaml | 14 + .../roles/vdi_tool/tasks/guacamole.yaml | 212 + .../vdi-setup/roles/vdi_tool/tasks/main.yaml | 26 + .../roles/vdi_tool/tasks/nomachine.yaml | 14 + .../roles/vdi_tool/tasks/workspot.yaml | 14 + .../templates/connection_bootstrap.sql.j2 | 72 + .../vdi_tool/templates/user_bootstrap.sql.j2 | 22 + .../vdi-setup/roles/vnc/tasks/main.yaml | 23 + .../vdi-setup/roles/vnc/tasks/tigervnc.yaml | 69 + .../vdi-setup/roles/vnc/tasks/tightvnc.yaml | 14 + .../roles/vnc/tasks/vnc_user_config.yaml | 59 + .../vdi-setup/roles/vnc/templates/config.j2 | 2 + .../roles/vnc/templates/vncserver.users.j2 | 3 + .../vnc/templates/vncserver@.service.rocky.j2 | 15 + .../templates/vncserver@.service.ubuntu.j2 | 16 + .../vdi-setup/templates/install.yaml.tftpl | 12 + .../vdi-setup/templates/vars.yaml.tftpl | 23 + .../modules/scripts/vdi-setup/validation.tf | 53 + .../modules/scripts/vdi-setup/variables.tf | 91 + .../modules/scripts/vdi-setup/versions.tf | 29 + .../scripts/wait-for-startup/README.md | 87 + .../modules/scripts/wait-for-startup/main.tf | 47 + .../scripts/wait-for-startup/metadata.yaml | 19 + .../scripts/wait-for-startup/outputs.tf | 15 + .../scripts/wait-for-startup-status.sh | 115 + .../scripts/wait-for-startup/variables.tf | 54 + .../scripts/wait-for-startup/versions.tf | 29 + .../scripts/windows-startup-script/README.md | 109 + .../scripts/windows-startup-script/main.tf | 34 + .../windows-startup-script/metadata.yaml | 18 + .../scripts/windows-startup-script/outputs.tf | 20 + .../templates/install_gpu_driver.ps1.tftpl | 38 + .../templates/setx_http_proxy.ps1 | 5 + .../windows-startup-script/variables.tf | 54 + .../windows-startup-script/versions.tf | 23 + .../modules/embedded/modules/README.md | 550 + .../compute/gke-job-template/README.md | 130 + .../modules/compute/gke-job-template/main.tf | 168 + .../compute/gke-job-template/metadata.yaml | 18 + .../compute/gke-job-template/outputs.tf | 27 + .../templates/gke-job-base.yaml.tftpl | 120 + .../compute/gke-job-template/variables.tf | 187 + .../compute/gke-job-template/versions.tf | 28 + .../modules/compute/gke-node-pool/README.md | 375 + .../compute/gke-node-pool/disk_definitions.tf | 38 + .../sample-tcpx-workload-job.yaml | 50 + .../sample-tcpxo-workload-job.yaml | 70 + .../scripts/enable-tcpx-in-workload.py | 185 + .../scripts/enable-tcpxo-in-workload.py | 186 + .../compute/gke-node-pool/gpu_direct.tf | 87 + .../compute/gke-node-pool/guest_cpus.tf | 32 + .../modules/compute/gke-node-pool/main.tf | 440 + .../compute/gke-node-pool/metadata.yaml | 21 + .../modules/compute/gke-node-pool/outputs.tf | 112 + .../gke-node-pool/reservation_definitions.tf | 99 + .../gke-node-pool/threads_per_core_calc.tf | 42 + .../compute/gke-node-pool/variables.tf | 472 + .../modules/compute/gke-node-pool/versions.tf | 34 + .../modules/compute/resource-policy/README.md | 82 + .../modules/compute/resource-policy/main.tf | 48 + .../compute/resource-policy/metadata.yaml | 19 + .../compute/resource-policy/outputs.tf | 28 + .../compute/resource-policy/variables.tf | 55 + .../compute/resource-policy/versions.tf | 34 + .../modules/compute/vm-instance/README.md | 258 + .../compute/vm-instance/compute_image.tf | 30 + .../modules/compute/vm-instance/main.tf | 349 + .../modules/compute/vm-instance/metadata.yaml | 19 + .../modules/compute/vm-instance/outputs.tf | 50 + .../startup_from_network_storage.tf | 65 + .../vm-instance/threads_per_core_calc.tf | 42 + .../modules/compute/vm-instance/variables.tf | 452 + .../modules/compute/vm-instance/versions.tf | 41 + .../modules/file-system/filestore/README.md | 248 + .../modules/file-system/filestore/main.tf | 116 + .../file-system/filestore/metadata.yaml | 19 + .../modules/file-system/filestore/outputs.tf | 62 + .../filestore/scripts/install-nfs-client.sh | 37 + .../file-system/filestore/scripts/mount.sh | 58 + .../file-system/filestore/scripts/mount.yaml | 39 + .../file-system/filestore/variables.tf | 189 + .../modules/file-system/filestore/versions.tf | 36 + .../gke-persistent-volume/README.md | 166 + .../file-system/gke-persistent-volume/main.tf | 124 + .../gke-persistent-volume/metadata.yaml | 18 + .../gke-persistent-volume/outputs.tf | 26 + .../templates/filestore-pv.yaml.tftpl | 23 + .../templates/filestore-pvc.yaml.tftpl | 17 + .../templates/gcs-pv.yaml.tftpl | 24 + .../templates/gcs-pvc.yaml.tftpl | 17 + .../gke-persistent-volume/variables.tf | 62 + .../gke-persistent-volume/versions.tf | 34 + .../modules/file-system/gke-storage/README.md | 133 + .../modules/file-system/gke-storage/main.tf | 78 + .../file-system/gke-storage/metadata.yaml | 18 + .../file-system/gke-storage/outputs.tf | 27 + .../hyperdisk-balanced-pvc.yaml.tftpl | 15 + .../hyperdisk-extreme-pvc.yaml.tftpl | 15 + .../hyperdisk-throughput-pvc.yaml.tftpl | 15 + .../parallelstore-pvc.yaml.tftpl | 15 + .../hyperdisk-balanced-sc.yaml.tftpl | 25 + .../hyperdisk-extreme-sc.yaml.tftpl | 24 + .../hyperdisk-throughput-sc.yaml.tftpl | 24 + .../storage-class/parallelstore-sc.yaml.tftpl | 21 + .../file-system/gke-storage/variables.tf | 138 + .../file-system/gke-storage/versions.tf | 21 + .../file-system/managed-lustre/README.md | 289 + .../file-system/managed-lustre/main.tf | 104 + .../file-system/managed-lustre/metadata.yaml | 19 + .../file-system/managed-lustre/outputs.tf | 43 + .../scripts/install-managed-lustre-client.sh | 84 + .../managed-lustre/scripts/mount.sh | 58 + .../file-system/managed-lustre/variables.tf | 131 + .../file-system/managed-lustre/versions.tf | 36 + .../file-system/parallelstore/README.md | 196 + .../modules/file-system/parallelstore/main.tf | 74 + .../file-system/parallelstore/metadata.yaml | 19 + .../file-system/parallelstore/outputs.tf | 47 + .../scripts/install-daos-client.sh | 112 + .../templates/mount-daos.sh.tftpl | 110 + .../file-system/parallelstore/variables.tf | 137 + .../file-system/parallelstore/versions.tf | 36 + .../pre-existing-network-storage/README.md | 192 + .../metadata.yaml | 18 + .../pre-existing-network-storage/outputs.tf | 124 + .../scripts/install-daos-client.sh | 112 + .../scripts/install-gcs-fuse.sh | 44 + .../scripts/install-managed-lustre-client.sh | 84 + .../scripts/install-nfs-client.sh | 37 + .../scripts/mount.sh | 58 + .../ddn_exascaler_luster_client_install.tftpl | 50 + .../templates/mount-daos.sh.tftpl | 110 + .../pre-existing-network-storage/variables.tf | 67 + .../pre-existing-network-storage/versions.tf | 19 + .../modules/internal/gpu-definition/README.md | 47 + .../modules/internal/gpu-definition/main.tf | 96 + .../internal/network-attachment/README.md | 54 + .../internal/network-attachment/main.tf | 70 + .../internal/network-attachment/metadata.yaml | 19 + .../modules/internal/vpc_peering/README.md | 56 + .../modules/internal/vpc_peering/main.tf | 80 + .../internal/vpc_peering/metadata.yaml | 19 + .../management/kubectl-apply/README.md | 171 + .../kubectl-apply/helm_install/README.md | 64 + .../kubectl-apply/helm_install/main.tf | 79 + .../kubectl-apply/helm_install/metadata.yaml | 19 + .../kubectl-apply/helm_install/variables.tf | 212 + .../kubectl-apply/helm_install/versions.tf | 24 + .../kubectl-apply/kubectl/README.md | 57 + .../management/kubectl-apply/kubectl/main.tf | 65 + .../kubectl-apply/kubectl/metadata.yaml | 19 + .../kubectl-apply/kubectl/variables.tf | 45 + .../kubectl-apply/kubectl/versions.tf | 30 + .../modules/management/kubectl-apply/main.tf | 234 + .../manifests/jobset-v0.5.2.yaml | 8349 ++++++++++ .../manifests/jobset-v0.7.2.yaml | 8918 ++++++++++ .../manifests/jobset-v0.8.1.yaml | 9075 ++++++++++ .../manifests/kueue-v0.10.0.yaml | 13185 +++++++++++++++ .../manifests/kueue-v0.10.1.yaml | 13175 +++++++++++++++ .../manifests/kueue-v0.11.4.yaml | 13466 +++++++++++++++ .../manifests/kueue-v0.12.2.yaml | 13851 ++++++++++++++++ .../management/kubectl-apply/metadata.yaml | 19 + .../management/kubectl-apply/providers.tf | 33 + .../management/kubectl-apply/variables.tf | 166 + .../management/kubectl-apply/versions.tf | 38 + .../modules/monitoring/dashboard/README.md | 86 + .../dashboard/dashboards/Empty.json.tpl | 17 + .../dashboard/dashboards/HPC.json.tpl | 595 + .../modules/monitoring/dashboard/main.tf | 35 + .../monitoring/dashboard/metadata.yaml | 19 + .../modules/monitoring/dashboard/outputs.tf | 23 + .../modules/monitoring/dashboard/variables.tf | 52 + .../modules/monitoring/dashboard/versions.tf | 29 + .../modules/network/firewall-rules/README.md | 107 + .../modules/network/firewall-rules/main.tf | 39 + .../network/firewall-rules/metadata.yaml | 19 + .../network/firewall-rules/variables.tf | 76 + .../network/firewall-rules/versions.tf | 29 + .../modules/network/gpu-rdma-vpc/README.md | 143 + .../modules/network/gpu-rdma-vpc/main.tf | 79 + .../network/gpu-rdma-vpc/metadata.yaml | 19 + .../modules/network/gpu-rdma-vpc/outputs.tf | 59 + .../modules/network/gpu-rdma-vpc/variables.tf | 164 + .../modules/network/gpu-rdma-vpc/versions.tf | 19 + .../modules/network/multivpc/README.md | 136 + .../embedded/modules/network/multivpc/main.tf | 78 + .../modules/network/multivpc/metadata.yaml | 19 + .../modules/network/multivpc/outputs.tf | 50 + .../modules/network/multivpc/variables.tf | 201 + .../modules/network/multivpc/versions.tf | 19 + .../network/pre-existing-subnetwork/README.md | 94 + .../network/pre-existing-subnetwork/main.tf | 38 + .../pre-existing-subnetwork/metadata.yaml | 21 + .../pre-existing-subnetwork/outputs.tf | 35 + .../pre-existing-subnetwork/variables.tf | 39 + .../pre-existing-subnetwork/versions.tf | 29 + .../network/pre-existing-vpc/README.md | 110 + .../modules/network/pre-existing-vpc/main.tf | 53 + .../network/pre-existing-vpc/metadata.yaml | 19 + .../network/pre-existing-vpc/outputs.tf | 50 + .../network/pre-existing-vpc/variables.tf | 37 + .../network/pre-existing-vpc/versions.tf | 29 + .../embedded/modules/network/vpc/README.md | 237 + .../embedded/modules/network/vpc/main.tf | 256 + .../modules/network/vpc/metadata.yaml | 19 + .../embedded/modules/network/vpc/outputs.tf | 68 + .../embedded/modules/network/vpc/variables.tf | 301 + .../embedded/modules/network/vpc/versions.tf | 19 + .../modules/packer/custom-image/README.md | 320 + .../modules/packer/custom-image/image.pkr.hcl | 216 + .../modules/packer/custom-image/metadata.yaml | 21 + .../packer/custom-image/variables.pkr.hcl | 276 + .../packer/custom-image/versions.pkr.hcl | 25 + .../scheduler/batch-job-template/README.md | 197 + .../batch-job-template/compute_image.tf | 30 + .../scheduler/batch-job-template/main.tf | 149 + .../batch-job-template/metadata.yaml | 22 + .../scheduler/batch-job-template/outputs.tf | 80 + .../startup_from_network_storage.tf | 65 + .../templates/batch-job-base.yaml.tftpl | 53 + .../templates/batch-submit.sh.tftpl | 10 + .../scheduler/batch-job-template/variables.tf | 240 + .../scheduler/batch-job-template/versions.tf | 37 + .../scheduler/batch-login-node/README.md | 127 + .../scheduler/batch-login-node/main.tf | 127 + .../scheduler/batch-login-node/metadata.yaml | 21 + .../scheduler/batch-login-node/outputs.tf | 37 + .../scheduler/batch-login-node/variables.tf | 151 + .../scheduler/batch-login-node/versions.tf | 29 + .../modules/scheduler/gke-cluster/README.md | 215 + .../modules/scheduler/gke-cluster/main.tf | 447 + .../scheduler/gke-cluster/metadata.yaml | 19 + .../modules/scheduler/gke-cluster/outputs.tf | 104 + .../templates/gke-network-paramset.yaml.tftpl | 9 + .../templates/network-object.yaml.tftpl | 11 + .../scheduler/gke-cluster/variables.tf | 521 + .../modules/scheduler/gke-cluster/versions.tf | 39 + .../pre-existing-gke-cluster/README.md | 116 + .../pre-existing-gke-cluster/main.tf | 70 + .../pre-existing-gke-cluster/metadata.yaml | 19 + .../pre-existing-gke-cluster/outputs.tf | 33 + .../templates/gke-network-paramset.yaml.tftpl | 9 + .../templates/network-object.yaml.tftpl | 11 + .../pre-existing-gke-cluster/variables.tf | 61 + .../pre-existing-gke-cluster/versions.tf | 30 + .../modules/scripts/startup-script/README.md | 348 + .../startup-script/files/configure-ssh.yml | 37 + .../startup-script/files/configure_proxy.sh | 54 + .../files/early_run_hotfixes.sh | 32 + .../startup-script/files/get_from_bucket.sh | 73 + .../startup-script/files/install_ansible.sh | 247 + .../files/install_cloud_rdma_drivers.sh | 38 + .../startup-script/files/install_docker.yml | 167 + .../files/install_gpu_network_wait_online.yml | 54 + .../files/install_managed_lustre.yml | 33 + .../files/install_monitoring_agent.sh | 144 + .../files/running-script-warning.sh | 26 + .../startup-script/files/setup-raid.yml | 100 + .../startup-script/files/setup-ssh-keys.sh | 19 + .../startup-script/files/setup-ssh-keys.yml | 40 + .../files/startup-script-stdlib-body.sh | 39 + .../files/startup-script-stdlib-head.sh | 266 + .../modules/scripts/startup-script/main.tf | 305 + .../scripts/startup-script/metadata.yaml | 19 + .../modules/scripts/startup-script/outputs.tf | 39 + .../templates/startup-script-custom.tftpl | 65 + .../scripts/startup-script/variables.tf | 298 + .../scripts/startup-script/versions.tf | 37 + vdi-test-scott/primary/providers.tf | 27 + vdi-test-scott/primary/variables.tf | 40 + vdi-test-scott/primary/versions.tf | 30 + 644 files changed, 140428 insertions(+), 232 deletions(-) create mode 100644 vdi-test-scott/.ghpc/artifacts/DO_NOT_MODIFY_THIS_DIRECTORY create mode 100644 vdi-test-scott/.ghpc/artifacts/expanded_blueprint.yaml create mode 100644 vdi-test-scott/.gitignore create mode 100644 vdi-test-scott/instructions.txt create mode 100644 vdi-test-scott/primary/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/validation.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/Intel-DAOS/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/scripts/mount.sh create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/providers.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/new-project/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/partition.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables_controller_instance.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles.tar.gz create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/debian.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/debian.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/rocky.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/ubuntu.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/rocky.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/defaults/main.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/os_login.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/nomachine.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/workspot.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/connection_bootstrap.sql.j2 create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/user_bootstrap.sql.j2 create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/tigervnc.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/tightvnc.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/vnc_user_config.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/config.j2 create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver.users.j2 create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.rocky.j2 create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.ubuntu.j2 create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/validation.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpx-in-workload.py create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/compute_image.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/mount.sh create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/mount.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-managed-lustre-client.sh create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/templates/mount-daos.sh.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/internal/gpu-definition/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/internal/gpu-definition/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/jobset-v0.5.2.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/jobset-v0.7.2.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/jobset-v0.8.1.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.10.0.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.10.1.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.12.2.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/providers.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/multivpc/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/multivpc/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/multivpc/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/multivpc/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/multivpc/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/multivpc/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/vpc/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/vpc/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/vpc/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/vpc/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/vpc/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/vpc/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/image.pkr.hcl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/variables.pkr.hcl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/versions.pkr.hcl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/README.md create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_docker.yml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/main.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/metadata.yaml create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/outputs.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/variables.tf create mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/versions.tf create mode 100644 vdi-test-scott/primary/providers.tf create mode 100644 vdi-test-scott/primary/variables.tf create mode 100644 vdi-test-scott/primary/versions.tf diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml index f148682825..3300920687 100644 --- a/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml @@ -46,24 +46,24 @@ {%- if is_nvidia_vws_supported -%} https://storage.googleapis.com/nvidia-drivers-us-public/GRID/vGPU18.1/NVIDIA-Linux-x86_64-570.133.20-grid.run {%- else -%} - + {%- endif %} # Include distribution-specific GPU driver setup - name: Include Ubuntu GPU driver setup ansible.builtin.include_tasks: gpu_driver_setup/ubuntu.yaml when: - - ansible_facts.distribution == 'Ubuntu' - - is_nvidia_vws_supported + - ansible_facts.distribution == 'Ubuntu' + - is_nvidia_vws_supported - name: Include Rocky Linux GPU driver setup ansible.builtin.include_tasks: gpu_driver_setup/rocky.yaml when: - - ansible_facts.distribution == 'Rocky' - - is_nvidia_vws_supported + - ansible_facts.distribution == 'Rocky' + - is_nvidia_vws_supported - name: Include Debian GPU driver setup ansible.builtin.include_tasks: gpu_driver_setup/debian.yaml when: - - ansible_facts.distribution == 'Debian' - - is_nvidia_vws_supported + - ansible_facts.distribution == 'Debian' + - is_nvidia_vws_supported diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/debian.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/debian.yaml index 7bd4f3699c..b99917397f 100644 --- a/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/debian.yaml +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/debian.yaml @@ -15,16 +15,16 @@ --- # Block for NVIDIA vWS driver setup (Debian) - block: - - name: Download NVIDIA vWS driver - ansible.builtin.uri: - url: "{{ driver_url }}" - method: GET - dest: /tmp/nvidia-driver.run - mode: '0755' - status_code: - - 200 - - 304 - - name: Install NVIDIA vWS driver (Debian) - ansible.builtin.command: - cmd: /tmp/nvidia-driver.run --dkms --silent - creates: /usr/bin/nvidia-smi + - name: Download NVIDIA vWS driver + ansible.builtin.uri: + url: "{{ driver_url }}" + method: GET + dest: /tmp/nvidia-driver.run + mode: '0755' + status_code: + - 200 + - 304 + - name: Install NVIDIA vWS driver (Debian) + ansible.builtin.command: + cmd: /tmp/nvidia-driver.run --dkms --silent + creates: /usr/bin/nvidia-smi diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/rocky.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/rocky.yaml index 260ca90d19..0c78bc330f 100644 --- a/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/rocky.yaml +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/rocky.yaml @@ -15,22 +15,22 @@ --- # Block for NVIDIA vWS driver setup (Rocky Linux) - block: - - name: Install NVIDIA Vulkan ICD loader (Rocky Linux) - ansible.builtin.package: - name: vulkan-loader - state: present - retries: 10 - delay: 3 - - name: Download NVIDIA vWS driver - ansible.builtin.uri: - url: "{{ driver_url }}" - method: GET - dest: /tmp/nvidia-driver.run - mode: '0755' - status_code: - - 200 - - 304 - - name: Install NVIDIA vWS driver (Rocky Linux) - ansible.builtin.command: - cmd: /tmp/nvidia-driver.run --dkms --silent - creates: /usr/bin/nvidia-smi + - name: Install NVIDIA Vulkan ICD loader (Rocky Linux) + ansible.builtin.package: + name: vulkan-loader + state: present + retries: 10 + delay: 3 + - name: Download NVIDIA vWS driver + ansible.builtin.uri: + url: "{{ driver_url }}" + method: GET + dest: /tmp/nvidia-driver.run + mode: '0755' + status_code: + - 200 + - 304 + - name: Install NVIDIA vWS driver (Rocky Linux) + ansible.builtin.command: + cmd: /tmp/nvidia-driver.run --dkms --silent + creates: /usr/bin/nvidia-smi diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/ubuntu.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/ubuntu.yaml index 8cef6585a0..fe7cae4439 100644 --- a/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/ubuntu.yaml +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/ubuntu.yaml @@ -15,59 +15,59 @@ --- # Block for NVIDIA vWS driver setup (Ubuntu) - block: - - name: Download NVIDIA vWS driver - ansible.builtin.uri: - url: "{{ driver_url }}" - method: GET - dest: /tmp/nvidia-driver.run - mode: '0755' - status_code: - - 200 - - 304 + - name: Download NVIDIA vWS driver + ansible.builtin.uri: + url: "{{ driver_url }}" + method: GET + dest: /tmp/nvidia-driver.run + mode: '0755' + status_code: + - 200 + - 304 - - name: Verify GCC version before driver installation - ansible.builtin.shell: gcc --version | head -1 - register: gcc_version_check - changed_when: false + - name: Verify GCC version before driver installation + ansible.builtin.shell: gcc --version | head -1 + register: gcc_version_check + changed_when: false - - name: Debug GCC version before driver installation - ansible.builtin.debug: - msg: "GCC version before driver installation: {{ gcc_version_check.stdout }}" + - name: Debug GCC version before driver installation + ansible.builtin.debug: + msg: "GCC version before driver installation: {{ gcc_version_check.stdout }}" - - name: Install NVIDIA vWS driver (Ubuntu) - ansible.builtin.command: - cmd: /tmp/nvidia-driver.run --dkms --silent - creates: /usr/bin/nvidia-smi - register: nvidia_install_result - failed_when: false + - name: Install NVIDIA vWS driver (Ubuntu) + ansible.builtin.command: + cmd: /tmp/nvidia-driver.run --dkms --silent + creates: /usr/bin/nvidia-smi + register: nvidia_install_result + failed_when: false - - name: Check if NVIDIA driver installation succeeded - ansible.builtin.stat: - path: /usr/bin/nvidia-smi - register: nvidia_smi_check + - name: Check if NVIDIA driver installation succeeded + ansible.builtin.stat: + path: /usr/bin/nvidia-smi + register: nvidia_smi_check - - name: Debug NVIDIA installation result - ansible.builtin.debug: - msg: - - "NVIDIA installer exit code: {{ nvidia_install_result.rc }}" - - "nvidia-smi exists: {{ nvidia_smi_check.stat.exists }}" - - "Installation stdout: {{ nvidia_install_result.stdout }}" - - "Installation stderr: {{ nvidia_install_result.stderr }}" + - name: Debug NVIDIA installation result + ansible.builtin.debug: + msg: + - "NVIDIA installer exit code: {{ nvidia_install_result.rc }}" + - "nvidia-smi exists: {{ nvidia_smi_check.stat.exists }}" + - "Installation stdout: {{ nvidia_install_result.stdout }}" + - "Installation stderr: {{ nvidia_install_result.stderr }}" - - name: Fail if NVIDIA driver installation failed - ansible.builtin.fail: - msg: "NVIDIA driver installation failed. Check /var/log/nvidia-installer.log for details." - when: - - nvidia_install_result.rc != 0 - - not nvidia_smi_check.stat.exists + - name: Fail if NVIDIA driver installation failed + ansible.builtin.fail: + msg: "NVIDIA driver installation failed. Check /var/log/nvidia-installer.log for details." + when: + - nvidia_install_result.rc != 0 + - not nvidia_smi_check.stat.exists - - name: Verify NVIDIA driver installation - ansible.builtin.command: nvidia-smi - register: nvidia_smi_test - changed_when: false - when: nvidia_smi_check.stat.exists + - name: Verify NVIDIA driver installation + ansible.builtin.command: nvidia-smi + register: nvidia_smi_test + changed_when: false + when: nvidia_smi_check.stat.exists - - name: Debug NVIDIA driver verification - ansible.builtin.debug: - msg: "NVIDIA driver verification: {{ nvidia_smi_test.stdout }}" - when: nvidia_smi_check.stat.exists + - name: Debug NVIDIA driver verification + ansible.builtin.debug: + msg: "NVIDIA driver verification: {{ nvidia_smi_test.stdout }}" + when: nvidia_smi_check.stat.exists diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml index 209155873b..982bae6c51 100644 --- a/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml @@ -45,9 +45,9 @@ - name: Ensure Python & tooling are installed ansible.builtin.package: name: - - python3 - - python3-pip - - jq + - python3 + - python3-pip + - jq state: present # GPU / vWS driver setup diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/rocky.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/rocky.yaml index 132e53fa89..c7d1b8c9a6 100644 --- a/community/modules/scripts/vdi-setup/roles/base_os/tasks/rocky.yaml +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/rocky.yaml @@ -15,7 +15,7 @@ --- - name: Set Rocky Linux specific variables ansible.builtin.set_fact: - gui_packages: ['@Xfce','@base-x'] + gui_packages: ['@Xfce', '@base-x'] vnc_pkg: >- {%- if vnc_flavor|lower == 'tigervnc' -%} tigervnc-server diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml index e467a0cca1..96d0370f7f 100644 --- a/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml @@ -53,8 +53,8 @@ - name: Debug kernel GCC version extraction ansible.builtin.debug: msg: - - "Raw kernel GCC version: {{ kernel_gcc_version.stdout }}" - - "Extracted required GCC version: {{ required_gcc_version }}" + - "Raw kernel GCC version: {{ kernel_gcc_version.stdout }}" + - "Extracted required GCC version: {{ required_gcc_version }}" - name: Get current GCC version ansible.builtin.shell: gcc --version | head -1 | grep -o '[0-9]*\.[0-9]*' | head -1 @@ -67,72 +67,72 @@ - name: Debug current GCC version extraction ansible.builtin.debug: - msg: - - "Raw current GCC version: {{ current_gcc_version.stdout }}" - - "Extracted current GCC major: {{ current_gcc_major }}" + msg: + - "Raw current GCC version: {{ current_gcc_version.stdout }}" + - "Extracted current GCC major: {{ current_gcc_major }}" - name: Debug GCC version information ansible.builtin.debug: - msg: - - "Kernel was built with GCC {{ required_gcc_version }}" - - "Current system GCC version: {{ current_gcc_major }}" - - "GCC update needed: {{ required_gcc_version != current_gcc_major }}" + msg: + - "Kernel was built with GCC {{ required_gcc_version }}" + - "Current system GCC version: {{ current_gcc_major }}" + - "GCC update needed: {{ required_gcc_version != current_gcc_major }}" - name: Install required GCC version if different from current block: - - name: Install GCC {{ required_gcc_version }} - ansible.builtin.package: - name: "gcc-{{ required_gcc_version }}" - state: present - retries: 10 - delay: 3 - - - name: Install G++ {{ required_gcc_version }} - ansible.builtin.package: - name: "g++-{{ required_gcc_version }}" - state: present - retries: 10 - delay: 3 - - - name: Set GCC {{ required_gcc_version }} as default - ansible.builtin.alternatives: - name: gcc - path: "/usr/bin/gcc-{{ required_gcc_version }}" - priority: "{{ required_gcc_version }}" - link: /usr/bin/gcc - - - name: Set G++ {{ required_gcc_version }} as default - ansible.builtin.alternatives: - name: g++ - path: "/usr/bin/g++-{{ required_gcc_version }}" - priority: "{{ required_gcc_version }}" - link: /usr/bin/g++ - - - name: Verify GCC version after update - ansible.builtin.shell: gcc --version | head -1 - register: updated_gcc_version - changed_when: false - - - name: Debug updated GCC version - ansible.builtin.debug: - msg: "Updated GCC version: {{ updated_gcc_version.stdout }}" - - - name: Verify G++ version after update - ansible.builtin.shell: g++ --version | head -1 - register: updated_gpp_version - changed_when: false - - - name: Debug updated G++ version - ansible.builtin.debug: - msg: "Updated G++ version: {{ updated_gpp_version.stdout }}" - - - name: Verify alternatives are set correctly - ansible.builtin.shell: update-alternatives --display gcc - register: gcc_alternatives_check - changed_when: false - - - name: Debug GCC alternatives status - ansible.builtin.debug: - msg: "GCC alternatives status: {{ gcc_alternatives_check.stdout }}" + - name: Install GCC {{ required_gcc_version }} + ansible.builtin.package: + name: "gcc-{{ required_gcc_version }}" + state: present + retries: 10 + delay: 3 + + - name: Install G++ {{ required_gcc_version }} + ansible.builtin.package: + name: "g++-{{ required_gcc_version }}" + state: present + retries: 10 + delay: 3 + + - name: Set GCC {{ required_gcc_version }} as default + ansible.builtin.alternatives: + name: gcc + path: "/usr/bin/gcc-{{ required_gcc_version }}" + priority: "{{ required_gcc_version }}" + link: /usr/bin/gcc + + - name: Set G++ {{ required_gcc_version }} as default + ansible.builtin.alternatives: + name: g++ + path: "/usr/bin/g++-{{ required_gcc_version }}" + priority: "{{ required_gcc_version }}" + link: /usr/bin/g++ + + - name: Verify GCC version after update + ansible.builtin.shell: gcc --version | head -1 + register: updated_gcc_version + changed_when: false + + - name: Debug updated GCC version + ansible.builtin.debug: + msg: "Updated GCC version: {{ updated_gcc_version.stdout }}" + + - name: Verify G++ version after update + ansible.builtin.shell: g++ --version | head -1 + register: updated_gpp_version + changed_when: false + + - name: Debug updated G++ version + ansible.builtin.debug: + msg: "Updated G++ version: {{ updated_gpp_version.stdout }}" + + - name: Verify alternatives are set correctly + ansible.builtin.shell: update-alternatives --display gcc + register: gcc_alternatives_check + changed_when: false + + - name: Debug GCC alternatives status + ansible.builtin.debug: + msg: "GCC alternatives status: {{ gcc_alternatives_check.stdout }}" when: required_gcc_version != current_gcc_major diff --git a/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml b/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml index 00e7fe540d..272e7163fd 100644 --- a/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml +++ b/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml @@ -17,8 +17,8 @@ set_fact: secret_name: "vdi-user-password-{{ item.username }}-{{ deployment_name }}" when: - - item.password is not defined - - item.secret_name is not defined + - item.password is not defined + - item.secret_name is not defined - name: Determine password source for user {{ item.username }} ansible.builtin.set_fact: @@ -117,8 +117,8 @@ item | combine({ 'password': vdiuser_password }) | combine( - (vnc_flavor | lower in ['tightvnc', 'tigervnc']) - | ternary({ + (vnc_flavor | lower in ['tightvnc', 'tigervnc']) + | ternary({ 'vncserver_password': lookup('community.general.random_string', length=8, special=false), 'display_number': item.port | int - 5900 }, {}) diff --git a/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml index 39ddf37c18..8eabc84b2c 100644 --- a/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml +++ b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml @@ -16,36 +16,36 @@ - name: Provision VDI users (custom or generated passwords) block: - # Create the Linux user with a hashed password - - name: Create VDI user {{ item.username }} - ansible.builtin.user: - name: "{{ item.username }}" - groups: "{{ vdi_user_group }},wheel" - shell: /bin/bash - create_home: yes - append: true - password: "{{ item.password | password_hash('sha512') }}" - update_password: on_create # sets pw if user is newly created + # Create the Linux user with a hashed password + - name: Create VDI user {{ item.username }} + ansible.builtin.user: + name: "{{ item.username }}" + groups: "{{ vdi_user_group }},wheel" + shell: /bin/bash + create_home: yes + append: true + password: "{{ item.password | password_hash('sha512') }}" + update_password: on_create # sets pw if user is newly created - # SSH key setup - - name: Ensure .ssh directory for {{ item.username }} - ansible.builtin.file: - path: "/home/{{ item.username }}/.ssh" - state: directory - owner: "{{ item.username }}" - group: "{{ vdi_user_group }}" - mode: '0700' + # SSH key setup + - name: Ensure .ssh directory for {{ item.username }} + ansible.builtin.file: + path: "/home/{{ item.username }}/.ssh" + state: directory + owner: "{{ item.username }}" + group: "{{ vdi_user_group }}" + mode: '0700' - - name: Generate SSH key pair for {{ item.username }} - community.crypto.openssh_keypair: - path: "/home/{{ item.username }}/.ssh/id_rsa" - owner: "{{ item.username }}" - group: "{{ vdi_user_group }}" - mode: '0600' - register: ssh_key + - name: Generate SSH key pair for {{ item.username }} + community.crypto.openssh_keypair: + path: "/home/{{ item.username }}/.ssh/id_rsa" + owner: "{{ item.username }}" + group: "{{ vdi_user_group }}" + mode: '0600' + register: ssh_key - - name: Add public key to authorized_keys for {{ item.username }} - ansible.builtin.authorized_key: - user: "{{ item.username }}" - key: "{{ ssh_key.public_key }}" - state: present + - name: Add public key to authorized_keys for {{ item.username }} + ansible.builtin.authorized_key: + user: "{{ item.username }}" + key: "{{ ssh_key.public_key }}" + state: present diff --git a/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml index 569db3dd6c..a8dc4f652c 100644 --- a/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml @@ -17,7 +17,7 @@ group: name: wheel state: present - + - name: Ensure VDI user group exists ansible.builtin.group: name: "{{ vdi_user_group }}" diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml index 4bb770a0c1..8aba005db8 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml @@ -21,8 +21,8 @@ group: root mode: '0755' loop: - - /opt/guacamole-db/data - - /opt/guacamole-db/initdb + - /opt/guacamole-db/data + - /opt/guacamole-db/initdb - name: Check Docker service status ansible.builtin.service_facts: @@ -110,16 +110,16 @@ - name: Pull required Docker images ansible.builtin.command: docker pull {{ item }} loop: - - postgres:latest - - guacamole/guacd:latest - - guacamole/guacamole:latest + - postgres:latest + - guacamole/guacd:latest + - guacamole/guacamole:latest - name: Create a dedicated Docker network ansible.builtin.command: docker network create guac_net register: network_create - failed_when: - - network_create.rc != 0 - - "'already exists' not in network_create.stderr" + failed_when: + - network_create.rc != 0 + - "'already exists' not in network_create.stderr" - name: Start Guacamole PostgreSQL container ansible.builtin.command: > @@ -133,9 +133,9 @@ --restart always postgres:latest register: postgres_start - failed_when: - - postgres_start.rc != 0 - - "'already in use' not in postgres_start.stderr" + failed_when: + - postgres_start.rc != 0 + - "'already in use' not in postgres_start.stderr" - name: Wait for PostgreSQL on host port ansible.builtin.wait_for: @@ -150,9 +150,9 @@ --restart always guacamole/guacd:latest register: guacd_start - failed_when: - - guacd_start.rc != 0 - - "'already in use' not in guacd_start.stderr" + failed_when: + - guacd_start.rc != 0 + - "'already in use' not in guacd_start.stderr" - name: Start Guacamole webapp container ansible.builtin.command: > @@ -168,9 +168,9 @@ --restart always guacamole/guacamole:latest register: guac_app_start - failed_when: - - guac_app_start.rc != 0 - - "'already in use' not in guac_app_start.stderr" + failed_when: + - guac_app_start.rc != 0 + - "'already in use' not in guac_app_start.stderr" - name: Wait for Guacamole HTTP endpoint ansible.builtin.wait_for: @@ -200,8 +200,8 @@ return_content: yes register: guac_api until: - - guac_api.status == 200 - - guac_api.json.authToken is defined + - guac_api.status == 200 + - guac_api.json.authToken is defined retries: 10 delay: 3 diff --git a/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml index bc81213782..1c1eadd0e3 100644 --- a/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml @@ -1,4 +1,3 @@ - # Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/modules/scripts/startup-script/files/install_docker.yml b/modules/scripts/startup-script/files/install_docker.yml index af251014b8..da096caf7d 100644 --- a/modules/scripts/startup-script/files/install_docker.yml +++ b/modules/scripts/startup-script/files/install_docker.yml @@ -34,8 +34,8 @@ state: present key: https://download.docker.com/linux/rhel/gpg when: - - is_rocky_linux - - not docker_binary.stat.exists + - is_rocky_linux + - not docker_binary.stat.exists register: add_docker_key retries: 30 delay: 10 @@ -45,32 +45,32 @@ dnf config-manager --add-repo https://download.docker.com/linux/rhel/docker-ce.repo when: - - is_rocky_linux - - not docker_binary.stat.exists + - is_rocky_linux + - not docker_binary.stat.exists notify: Refresh dnf cache - name: Install Docker Engine and related packages on Rocky ansible.builtin.package: name: - - docker-ce - - docker-ce-cli - - containerd.io - - docker-buildx-plugin - - docker-compose-plugin + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin state: present update_cache: yes when: - - is_rocky_linux - - not docker_binary.stat.exists + - is_rocky_linux + - not docker_binary.stat.exists - name: Ensure Docker SDK for Python is installed block: - - name: Install python3-pip - ansible.builtin.package: - name: python3-pip - state: present - - name: Install Docker SDK for Python - ansible.builtin.pip: - name: docker - executable: /usr/local/ghpc-venv/bin/pip3 + - name: Install python3-pip + ansible.builtin.package: + name: python3-pip + state: present + - name: Install Docker SDK for Python + ansible.builtin.pip: + name: docker + executable: /usr/local/ghpc-venv/bin/pip3 when: is_rocky_linux - name: Download Docker Installer (get.docker.com) ansible.builtin.get_url: @@ -80,15 +80,15 @@ group: root mode: '0644' when: - - not is_rocky_linux - - not docker_binary.stat.exists + - not is_rocky_linux + - not docker_binary.stat.exists - name: Install Docker ansible.builtin.command: sh /tmp/get-docker.sh register: docker_installed changed_when: docker_installed.rc != 0 when: - - not is_rocky_linux - - not docker_binary.stat.exists + - not is_rocky_linux + - not docker_binary.stat.exists - name: Create Docker daemon configuration ansible.builtin.copy: dest: /etc/docker/daemon.json diff --git a/vdi-test-scott/.ghpc/artifacts/DO_NOT_MODIFY_THIS_DIRECTORY b/vdi-test-scott/.ghpc/artifacts/DO_NOT_MODIFY_THIS_DIRECTORY new file mode 100644 index 0000000000..56f49c329a --- /dev/null +++ b/vdi-test-scott/.ghpc/artifacts/DO_NOT_MODIFY_THIS_DIRECTORY @@ -0,0 +1 @@ +Files in this directory are managed by gcluster. Do not modify them manually! diff --git a/vdi-test-scott/.ghpc/artifacts/expanded_blueprint.yaml b/vdi-test-scott/.ghpc/artifacts/expanded_blueprint.yaml new file mode 100644 index 0000000000..eac3bcfc02 --- /dev/null +++ b/vdi-test-scott/.ghpc/artifacts/expanded_blueprint.yaml @@ -0,0 +1,112 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +blueprint_name: vdi-test-scott +ghpc_version: v1.57.2-80-ge3e815e7a +vars: + deployment_name: vdi-test-scott + labels: + ghpc_blueprint: vdi-test-scott + ghpc_deployment: ((var.deployment_name)) + project_id: hpc-discovery-external + region: us-central1 + zone: us-central1-a +deployment_groups: + - group: primary + terraform_providers: + google: + source: hashicorp/google + version: ~> 6.41.0 + configuration: + project: ((var.project_id)) + region: ((var.region)) + zone: ((var.zone)) + google-beta: + source: hashicorp/google-beta + version: ~> 6.41.0 + configuration: + project: ((var.project_id)) + region: ((var.region)) + zone: ((var.zone)) + modules: + - source: modules/network/vpc + kind: terraform + id: network1 + settings: + deployment_name: ((var.deployment_name)) + extra_iap_ports: + - 8080 + firewall_rules: + - allow: + - ports: + - "8080" + protocol: tcp + description: Allow external ingress to Guacamole on TCP port 8080 + direction: INGRESS + name: allow-guacamole-8080-ext + ranges: + - 0.0.0.0/0 + labels: ((var.labels)) + project_id: ((var.project_id)) + region: ((var.region)) + - source: community/modules/project/service-enablement + kind: terraform + id: enable-apis + settings: + gcp_service_list: + - secretmanager.googleapis.com + - storage.googleapis.com + - compute.googleapis.com + project_id: ((var.project_id)) + - source: community/modules/scripts/vdi-setup + kind: terraform + id: vdi-setup + settings: + deployment_name: ((var.deployment_name)) + labels: ((var.labels)) + project_id: ((var.project_id)) + region: ((var.region)) + user_provision: local_users + vdi_resolution: 1920x1080 + vdi_tool: guacamole + vdi_user_group: vdiusers + vdi_users: + - port: 5901 + username: alice + - port: 5902 + secret_name: a-password-for-bob + username: bob + vnc_flavor: tigervnc + - source: modules/compute/vm-instance + kind: terraform + id: machine + use: + - network1 + - vdi-setup + settings: + deployment_name: ((var.deployment_name)) + disable_public_ips: false + instance_image: + family: debian-11 + project: debian-cloud + labels: ((var.labels)) + machine_type: g2-standard-8 + network_self_link: ((module.network1.network_self_link)) + project_id: ((var.project_id)) + region: ((var.region)) + startup_script: ((module.vdi-setup.startup_script)) + subnetwork_self_link: ((module.network1.subnetwork_self_link)) + tags: + - guacamole + zone: ((var.zone)) diff --git a/vdi-test-scott/.gitignore b/vdi-test-scott/.gitignore new file mode 100644 index 0000000000..1e44b25074 --- /dev/null +++ b/vdi-test-scott/.gitignore @@ -0,0 +1,48 @@ +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc + +# Cache objects +packer_cache/ + +# https://www.packer.io/guides/hcl/variables +# Exclude all .pkrvars.hcl files, which are likely to contain sensitive data, +# such as password, private keys, and other secrets. These should not be part of +# version control as they are data points which are potentially sensitive and +# subject to change depending on the environment. +# +*.pkrvars.hcl + +# For built boxes +*.box diff --git a/vdi-test-scott/instructions.txt b/vdi-test-scott/instructions.txt new file mode 100644 index 0000000000..4d37623497 --- /dev/null +++ b/vdi-test-scott/instructions.txt @@ -0,0 +1,23 @@ +Advanced Deployment Instructions +================================ + +Terraform group 'primary' was successfully created in directory vdi-test-scott/primary +To deploy, run the following commands: + +terraform -chdir=vdi-test-scott/primary init +terraform -chdir=vdi-test-scott/primary validate +terraform -chdir=vdi-test-scott/primary apply + +Destroying infrastructure when no longer needed +=============================================== + +Automated +--------- + +gcluster destroy vdi-test-scott + +Advanced / Manual +----------------- +Infrastructure should be destroyed in reverse order of creation: + +terraform -chdir=vdi-test-scott/primary destroy diff --git a/vdi-test-scott/primary/main.tf b/vdi-test-scott/primary/main.tf new file mode 100644 index 0000000000..ff7414b420 --- /dev/null +++ b/vdi-test-scott/primary/main.tf @@ -0,0 +1,80 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "network1" { + source = "./modules/embedded/modules/network/vpc" + deployment_name = var.deployment_name + extra_iap_ports = [8080] + firewall_rules = [{ + allow = [{ + ports = ["8080"] + protocol = "tcp" + }] + description = "Allow external ingress to Guacamole on TCP port 8080" + direction = "INGRESS" + name = "allow-guacamole-8080-ext" + ranges = ["0.0.0.0/0"] + }] + labels = var.labels + project_id = var.project_id + region = var.region +} + +module "enable-apis" { + source = "./modules/embedded/community/modules/project/service-enablement" + gcp_service_list = ["secretmanager.googleapis.com", "storage.googleapis.com", "compute.googleapis.com"] + project_id = var.project_id +} + +module "vdi-setup" { + source = "./modules/embedded/community/modules/scripts/vdi-setup" + deployment_name = var.deployment_name + labels = var.labels + project_id = var.project_id + region = var.region + user_provision = "local_users" + vdi_resolution = "1920x1080" + vdi_tool = "guacamole" + vdi_user_group = "vdiusers" + vdi_users = [{ + port = 5901 + username = "alice" + }, { + port = 5902 + secret_name = "a-password-for-bob" + username = "bob" + }] + vnc_flavor = "tigervnc" +} + +module "machine" { + source = "./modules/embedded/modules/compute/vm-instance" + deployment_name = var.deployment_name + disable_public_ips = false + instance_image = { + family = "debian-11" + project = "debian-cloud" + } + labels = var.labels + machine_type = "g2-standard-8" + network_self_link = module.network1.network_self_link + project_id = var.project_id + region = var.region + startup_script = module.vdi-setup.startup_script + subnetwork_self_link = module.network1.subnetwork_self_link + tags = ["guacamole"] + zone = var.zone +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/README.md new file mode 100644 index 0000000000..0c83b9d30c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/README.md @@ -0,0 +1,7 @@ +# Community Modules + +This directory contains modules that rely on partner resources, have been +contributed by outside developers or are in early development by the Cluster Toolkit +team. The modules in this directory are listed alongside core modules in the +[core modules README](../../modules/README.md). There you can also learn more +about general use and how to write custom Cluster Toolkit modules. diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/README.md new file mode 100644 index 0000000000..4f65411ddf --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/README.md @@ -0,0 +1,271 @@ +## Description + +This module performs the following tasks: + +- create an instance template from which execute points will be created +- create a managed instance group ([MIG][mig]) for execute points +- create a Toolkit runner to configure the autoscaler to scale the MIG + +It is expected to be used with the [htcondor-install] and [htcondor-setup] +modules. + +[htcondor-install]: ../../scripts/htcondor-install/README.md +[htcondor-setup]: ../../scheduler/htcondor-setup/README.md +[mig]: https://cloud.google.com/compute/docs/instance-groups/ + +### Known limitations + +This module may be used multiple times in a blueprint to create sets of +execute points in an HTCondor pool. If used more than 1 time, the setting +[name_prefix](#input_name_prefix) must be set to a value that is unique across +all uses of the htcondor-execute-point module. If you do not follow this +constraint, you will likely receive an error while running `terraform apply` +similar to that shown below. + +```text +Error: Invalid value for variable + + on modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf line 136, in module "startup_script": + 136: runners = local.all_runners + ā”œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + ā”‚ var.runners is list of map of string with 5 elements + +All startup-script runners must have a unique destination. +``` + +### How to configure jobs to select execute points + +HTCondor access points provisioned by the Toolkit are specially configured to +honor an attribute named `RequireId` in each [Job ClassAd][jobad]. This value +must be set to the ID of a MIG created by an instance of this module. The +[htcondor-access-point] module includes a setting `var.default_mig_id` that will +set this value automatically to the MIG ID corresponding to the module's +execute points. If this setting is left unset each job must specify `+RequireId` +explicitly. In all cases, the default value can be overridden explicitly as shown +below: + +```text +universe = vanilla +executable = /bin/echo +arguments = "Hello, World!" +output = out.$(ClusterId).$(ProcId) +error = err.$(ClusterId).$(ProcId) +log = log.$(ClusterId).$(ProcId) +request_cpus = 1 +request_memory = 100MB ++RequireId = "htcondor-pool-ep-mig" +queue +``` + +[htcondor-access-point]: ../../scheduler/htcondor-access-point/README.md +[jobad]: https://htcondor.readthedocs.io/en/latest/users-manual/matchmaking-with-classads.html + +### Example + +A full example can be found in the [examples README][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- + +The following code snippet creates a pool with 2 sets of HTCondor execute +points, one using On-demand pricing and the other using Spot pricing. They use +a startup script and network created in previous steps. + +```yaml +- id: htcondor_execute_point + source: community/modules/compute/htcondor-execute-point + use: + - network1 + - htcondor_secrets + - htcondor_setup + - htcondor_cm + settings: + instance_image: + project: $(vars.project_id) + family: $(vars.new_image_family) + min_idle: 2 + +- id: htcondor_execute_point_spot + source: community/modules/compute/htcondor-execute-point + use: + - network1 + - htcondor_secrets + - htcondor_setup + - htcondor_cm + settings: + instance_image: + project: $(vars.project_id) + family: $(vars.new_image_family) + spot: true + +- id: htcondor_access + source: community/modules/scheduler/htcondor-access-point + use: + - network1 + - htcondor_secrets + - htcondor_setup + - htcondor_cm + - htcondor_execute_point + - htcondor_execute_point_spot + settings: + default_mig_id: $(htcondor_execute_point.mig_id) + enable_public_ips: true + instance_image: + project: $(vars.project_id) + family: $(vars.new_image_family) + outputs: + - access_point_ips + - access_point_name +``` + +## Support + +HTCondor is maintained by the [Center for High Throughput Computing][chtc] at +the University of Wisconsin-Madison. Support for HTCondor is available via: + +- [Discussion lists](https://htcondor.org/mail-lists/) +- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) +- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) + +[chtc]: https://chtc.cs.wisc.edu/ + +## Behavior of Managed Instance Group (MIG) + +Regional [MIGs][mig] are used to provision Execute Points. By default, VMs +will be provisioned in any of the zones available in that region, however, it +can be constrained to run in fewer zones (or a single zone) using +[var.zones](#input_zones). + +When the configuration of an Execute Point is changed, the MIG can be configured +to [replace the VM][replacement] using a "proactive" or "opportunistic" policy. +By default, the policy is set to opportunistic. In practice, this means that +Execute Points will _NOT_ be automatically replaced by Terraform when changes to +the instance template / HTCondor configuration are made. We recommend leaving +this at the default value as it will allow the HTCondor autoscaler to replace +VMs when they become idle without disrupting running jobs. + +However, if it is desired [var.update_policy](#input_update_policy) can be set +to "PROACTIVE" to enable automatic replacement. This will disrupt running jobs +and send them back to the queue. Alternatively, one can leave the setting at +the default value of "OPPORTUNISTIC" and update: + +- intentionally by issuing an update via Cloud Console or using gcloud (below) +- VMs becomes unhealthy or are otherwise automatically replaced (e.g. regular + Google Cloud maintenance) + +For example, to manually update all instances in a MIG: + +```text +gcloud compute instance-groups managed update-instances \ + <> --all-instances --region <> \ + --project <> --minimal-action replace +``` + +[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type + +## Known Issues + +When using OS Login with "external users" (outside of the Google Cloud +organization), then Docker universe jobs will fail and cause the Docker daemon +to crash. This stems from the use of POSIX user ids (uid) outside the range +supported by Docker. Please consider disabling OS Login if this atypical +situation applies. + +```yaml +vars: + # add setting below to existing deployment variables + enable_oslogin: DISABLE +``` + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.1 | +| [google](#requirement\_google) | >= 4.0 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.0 | +| [null](#provider\_null) | >= 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [execute\_point\_instance\_template](#module\_execute\_point\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | +| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | +| [mig](#module\_mig) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.execute_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [null_resource.execute_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [central\_manager\_ips](#input\_central\_manager\_ips) | List of IP addresses of HTCondor Central Managers | `list(string)` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `100` | no | +| [disk\_type](#input\_disk\_type) | Disk type for template | `string` | `"pd-balanced"` | no | +| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape across zones for instance group managing execute points | `string` | `"ANY"` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | +| [execute\_point\_runner](#input\_execute\_point\_runner) | A list of Toolkit runners for configuring an HTCondor execute point | `list(map(string))` | `[]` | no | +| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | Service account for HTCondor execute point (e-mail format) | `string` | n/a | yes | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | +| [instance\_image](#input\_instance\_image) | HTCondor execute point VM image

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | +| [labels](#input\_labels) | Labels to add to HTConodr execute points | `map(string)` | n/a | yes | +| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor execute points | `string` | `"n2-standard-4"` | no | +| [max\_size](#input\_max\_size) | Maximum size of the HTCondor execute point pool. | `number` | `5` | no | +| [metadata](#input\_metadata) | Metadata to add to HTCondor execute points | `map(string)` | `{}` | no | +| [min\_idle](#input\_min\_idle) | Minimum number of idle VMs in the HTCondor pool (if pool reaches var.max\_size, this minimum is not guaranteed); set to ensure jobs beginning run more quickly. | `number` | `0` | no | +| [name\_prefix](#input\_name\_prefix) | Name prefix given to hostnames in this group of execute points; must be unique across all instances of this module | `string` | n/a | yes | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network HTCondor execute points will join | `string` | `"default"` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | Project in which the HTCondor execute points will be created | `string` | n/a | yes | +| [region](#input\_region) | The region in which HTCondor execute points will be created | `string` | n/a | yes | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork HTCondor execute points will join | `string` | `null` | no | +| [target\_size](#input\_target\_size) | Initial size of the HTCondor execute point pool; set to null (default) to avoid Terraform management of size. | `number` | `null` | no | +| [update\_policy](#input\_update\_policy) | Replacement policy for Access Point Managed Instance Group ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle) | `string` | `"OPPORTUNISTIC"` | no | +| [windows\_startup\_ps1](#input\_windows\_startup\_ps1) | Startup script to run at boot-time for Windows-based HTCondor execute points | `list(string)` | `[]` | no | +| [zones](#input\_zones) | Zone(s) in which execute points may be created. If not supplied, will default to all zones in var.region. | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [autoscaler\_runner](#output\_autoscaler\_runner) | Toolkit runner to configure the HTCondor autoscaler | +| [mig\_id](#output\_mig\_id) | ID of the managed instance group containing the execute points | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf new file mode 100644 index 0000000000..7a7fe02307 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_compute_image" "compute_image" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = try(var.instance_image.project, null) + + lifecycle { + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml new file mode 100644 index 0000000000..375ae036cd --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml @@ -0,0 +1,74 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure HTCondor Role + hosts: localhost + become: true + vars: + spool_dir: /var/lib/condor/spool + condor_config_root: /etc/condor + ghpc_config_file: 50-ghpc-managed + tasks: + - name: Ensure necessary variables are set + ansible.builtin.assert: + that: + - htcondor_role is defined + - config_object is defined + - name: Remove default HTCondor configuration + ansible.builtin.file: + path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" + state: absent + notify: + - Reload HTCondor + - name: Create Toolkit configuration file + register: config_update + changed_when: config_update.rc == 137 + failed_when: config_update.rc != 0 and config_update.rc != 137 + ansible.builtin.shell: | + set -e -o pipefail + REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) + + CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" + if [ -f "${CONFIG_FILE}" ]; then + LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") + else + LOCAL_HASH="INVALID-HASH" + fi + + if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then + gcloud storage cp {{ config_object }} "${CONFIG_FILE}" + chmod 0644 "${CONFIG_FILE}" + exit 137 + fi + args: + executable: /bin/bash + notify: + - Reload HTCondor + handlers: + - name: Reload HTCondor + ansible.builtin.service: + name: condor + state: reloaded + post_tasks: + - name: Start HTCondor + ansible.builtin.service: + name: condor + state: started + enabled: true + - name: Inform users + changed_when: false + ansible.builtin.shell: | + set -e -o pipefail + wall "******* HTCondor system configuration complete ********" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml new file mode 100644 index 0000000000..a85158fdfc --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml @@ -0,0 +1,98 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This playbook makes the assumption that a virtual environment has been created +# with the autoscaler and its dependencies previously installed. A runner that +# does this is provided as an output of the htcondor-install module within the +# Cluster Toolkit at community/modules/scripts/htcondor-install. + +--- +- name: Configure HTCondor Autoscaler + hosts: all + vars: + python: /usr/local/htcondor/bin/python3 + autoscaler: /usr/local/htcondor/bin/autoscaler.py + systemd_override_path: /etc/systemd/system + become: true + tasks: + - name: User must supply HTCondor role + ansible.builtin.assert: + that: + - project_id is defined + - region is defined + - zone is defined + - mig_id is defined + - max_size is defined + - name: Create SystemD service for HTCondor autoscaler + ansible.builtin.copy: + dest: "{{ systemd_override_path }}/htcondor-autoscaler@.service" + mode: 0644 + content: | + [Unit] + Description=HTCondor Autoscaler MIG: %i + + [Service] + User=condor + Type=oneshot + ExecStart={{ python }} {{ autoscaler }} --p $PROJECT_ID --r $REGION --z $ZONE --mz --g %i --c $MAX_SIZE --i $MIN_IDLE + notify: + - Reload SystemD + - name: Create SystemD override directory for autoscaler configuration + ansible.builtin.file: + path: "{{ systemd_override_path }}/htcondor-autoscaler@{{ mig_id }}.service.d" + state: directory + owner: root + group: root + mode: 0755 + - name: Create autoscaler configuration + ansible.builtin.copy: + dest: "{{ systemd_override_path }}/htcondor-autoscaler@{{ mig_id }}.service.d/miglimit.conf" + mode: 0644 + content: | + [Service] + Environment=PROJECT_ID={{ project_id }} + Environment=REGION={{ region }} + Environment=ZONE={{ zone }} + Environment=MAX_SIZE={{ max_size }} + Environment=MIN_IDLE={{ min_idle }} + notify: + - Reload SystemD + - name: Create SystemD timer for HTCondor autoscaler + ansible.builtin.copy: + dest: "{{ systemd_override_path }}/htcondor-autoscaler@.timer" + mode: 0644 + content: | + [Unit] + Description=Run HTCondor Autoscaler Periodically + + [Timer] + OnCalendar=minutely + AccuracySec=1us + RandomizedDelaySec=30 + # the directive below is ignored harmlessly on CentOS 7; this has impact + # that timing averages to 1 minute but is not precisely 1 minute; still + # useful to ensure that timers for different MIGs do not overlap + FixedRandomDelay=true + notify: + - Reload SystemD + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true + post_tasks: + - name: Activate HTCondor Autoscaler timer + ansible.builtin.systemd: + name: htcondor-autoscaler@{{ mig_id }}.timer + enabled: true + state: started diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf new file mode 100644 index 0000000000..7b0df94987 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf @@ -0,0 +1,218 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "htcondor-execute-point", ghpc_role = "compute" }) +} + +module "gpu" { + source = "../../../../modules/internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + zones = coalescelist(var.zones, data.google_compute_zones.available.names) + network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } + + oslogin_api_values = { + "DISABLE" = "FALSE" + "ENABLE" = "TRUE" + } + enable_oslogin = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } + + windows_startup_ps1 = join("\n\n", flatten([var.windows_startup_ps1, local.execute_config_windows_startup_ps1])) + + is_windows_image = anytrue([for l in data.google_compute_image.compute_image.licenses : length(regexall("windows-cloud", l)) > 0]) + windows_startup_metadata = local.is_windows_image && local.windows_startup_ps1 != "" ? { + windows-startup-script-ps1 = local.windows_startup_ps1 + } : {} + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + metadata = merge( + local.windows_startup_metadata, + local.network_storage_metadata, + local.enable_oslogin, + local.disable_automatic_updates_metadata, + var.metadata + ) + + autoscaler_runner = { + "type" = "ansible-local" + "content" = file("${path.module}/files/htcondor_configure_autoscaler.yml") + "destination" = "htcondor_configure_autoscaler_${module.mig.instance_group_manager.name}.yml" + "args" = join(" ", [ + "-e project_id=${var.project_id}", + "-e region=${var.region}", + "-e zone=${local.zones[0]}", # this value is required, but ignored by regional MIG autoscaler + "-e mig_id=${module.mig.instance_group_manager.name}", + "-e max_size=${var.max_size}", + "-e min_idle=${var.min_idle}", + ]) + } + + execute_config = templatefile("${path.module}/templates/condor_config.tftpl", { + htcondor_role = "get_htcondor_execute", + central_manager_ips = var.central_manager_ips, + guest_accelerator = local.guest_accelerator, + }) + + execute_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.execute_config.output_name}" + execute_runner = { + type = "ansible-local" + content = file("${path.module}/files/htcondor_configure.yml") + destination = "htcondor_configure.yml" + args = join(" ", [ + "-e htcondor_role=get_htcondor_execute", + "-e config_object=${local.execute_object}", + ]) + } + + native_fstype = [] + startup_script_network_storage = [ + for ns in var.network_storage : + ns if !contains(local.native_fstype, ns.fs_type) + ] + storage_client_install_runners = [ + for ns in local.startup_script_network_storage : + ns.client_install_runner if ns.client_install_runner != null + ] + mount_runners = [ + for ns in local.startup_script_network_storage : + ns.mount_runner if ns.mount_runner != null + ] + + all_runners = concat( + local.storage_client_install_runners, + local.mount_runners, + var.execute_point_runner, + [local.execute_runner], + ) + + execute_config_windows_startup_ps1 = templatefile( + "${path.module}/templates/download-condor-config.ps1.tftpl", + { + config_object = local.execute_object, + } + ) + + name_prefix = "${var.deployment_name}-${var.name_prefix}-ep" +} + +data "google_compute_zones" "available" { + project = var.project_id + region = var.region +} + +resource "null_resource" "execute_config" { + triggers = { + config = local.execute_config + } +} + +resource "google_storage_bucket_object" "execute_config" { + name = "${local.name_prefix}-config-${substr(md5(null_resource.execute_config.id), 0, 4)}" + content = local.execute_config + bucket = var.htcondor_bucket_name +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + project_id = var.project_id + region = var.region + labels = local.labels + deployment_name = var.deployment_name + + runners = local.all_runners +} + +module "execute_point_instance_template" { + source = "terraform-google-modules/vm/google//modules/instance_template" + version = "~> 12.1" + + name_prefix = local.name_prefix + project_id = var.project_id + network = var.network_self_link + subnetwork = var.subnetwork_self_link + service_account = { + email = var.execute_point_service_account_email + scopes = var.service_account_scopes + } + labels = local.labels + + machine_type = var.machine_type + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + gpu = one(local.guest_accelerator) + preemptible = var.spot + startup_script = local.is_windows_image ? null : module.startup_script.startup_script + metadata = local.metadata + source_image = data.google_compute_image.compute_image.self_link + + # secure boot + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config +} + +module "mig" { + source = "terraform-google-modules/vm/google//modules/mig" + version = "~> 12.1" + + project_id = var.project_id + region = var.region + distribution_policy_target_shape = var.distribution_policy_target_shape + distribution_policy_zones = local.zones + target_size = var.target_size + hostname = local.name_prefix + mig_name = local.name_prefix + instance_template = module.execute_point_instance_template.self_link + + health_check_name = "health-htcondor-${local.name_prefix}" + health_check = { + type = "tcp" + initial_delay_sec = 600 + check_interval_sec = 20 + healthy_threshold = 2 + timeout_sec = 8 + unhealthy_threshold = 3 + response = "" + proxy_header = "NONE" + port = 9618 + request = "" + request_path = "" + host = "" + enable_logging = true + } + + update_policy = [{ + instance_redistribution_type = "NONE" + replacement_method = "SUBSTITUTE" + max_surge_fixed = length(local.zones) + max_unavailable_fixed = length(local.zones) + max_surge_percent = null + max_unavailable_percent = null + min_ready_sec = 300 + minimal_action = "REPLACE" + type = var.update_policy + }] + +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml new file mode 100644 index 0000000000..3a78f9a46b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - storage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf new file mode 100644 index 0000000000..b31f40130f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "autoscaler_runner" { + value = local.autoscaler_runner + description = "Toolkit runner to configure the HTCondor autoscaler" +} + +output "mig_id" { + value = module.mig.instance_group_manager.name + description = "ID of the managed instance group containing the execute points" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl new file mode 100644 index 0000000000..c8f5ce31a8 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl @@ -0,0 +1,31 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this file is managed by the Cluster Toolkit; do not edit it manually +# override settings with a higher priority (last lexically) named file +# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration + +use role:${htcondor_role} +CONDOR_HOST = ${join(",", central_manager_ips)} + +# StartD configuration settings +%{ if length(guest_accelerator) > 0 ~} +use feature:GPUs +%{ endif ~} +use feature:PartitionableSlot +use feature:CommonCloudAttributesGoogle("-c created-by") +UPDATE_INTERVAL = 30 +TRUST_UID_DOMAIN = True +STARTER_ALLOW_RUNAS_OWNER = True +RUNBENCHMARKS = False diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl new file mode 100644 index 0000000000..19789f122e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl @@ -0,0 +1,34 @@ +# create directory for local condor_config customizations +$config_dir = 'C:\Condor\config' +if(!(test-path -PathType container -Path $config_dir)) +{ + New-Item -ItemType Directory -Path $config_dir +} + +# update local condor_config if blueprint has changed +$config_file = "$config_dir\50-ghpc-managed" +if (Test-Path -Path $config_file -PathType Leaf) +{ + $local_hash = gcloud --format="value(md5_hash)" storage hash $config_file +} +else +{ + $local_hash = "INVALID-HASH" +} + +$remote_hash = gcloud --format="value(md5_hash)" storage hash ${config_object} +if ($local_hash -cne $remote_hash) +{ + Write-Output "Updating condor configuration" + gcloud storage cp ${config_object} $config_file + if ($LASTEXITCODE -ne 0) + { + throw "Could not download HTCondor configuration; exiting startup script" + } + Restart-Service condor +} + +# ignored if service is already running; must be here to handle case where +# machine is rebooted, but configuration has previously been downloaded +# and service is disabled from automatic start +Start-Service condor diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf new file mode 100644 index 0000000000..aab8a54c2d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf @@ -0,0 +1,265 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HTCondor execute points will be created" + type = string +} + +variable "region" { + description = "The region in which HTCondor execute points will be created" + type = string +} + +variable "zones" { + description = "Zone(s) in which execute points may be created. If not supplied, will default to all zones in var.region." + type = list(string) + default = [] + nullable = false +} + +variable "distribution_policy_target_shape" { + description = "Target shape across zones for instance group managing execute points" + type = string + default = "ANY" +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "labels" { + description = "Labels to add to HTConodr execute points" + type = map(string) +} + +variable "machine_type" { + description = "Machine type to use for HTCondor execute points" + type = string + default = "n2-standard-4" +} + +variable "execute_point_runner" { + description = "A list of Toolkit runners for configuring an HTCondor execute point" + type = list(map(string)) + default = [] +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured" + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "instance_image" { + description = <<-EOD + HTCondor execute point VM image + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + default = { + project = "cloud-hpc-image-public" + family = "hpc-rocky-linux-8" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "execute_point_service_account_email" { + description = "Service account for HTCondor execute point (e-mail format)" + type = string +} + +variable "service_account_scopes" { + description = "Scopes by which to limit service account attached to central manager." + type = set(string) + default = [ + "https://www.googleapis.com/auth/cloud-platform", + ] +} + +variable "network_self_link" { + description = "The self link of the network HTCondor execute points will join" + type = string + default = "default" +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork HTCondor execute points will join" + type = string + default = null +} + +variable "target_size" { + description = "Initial size of the HTCondor execute point pool; set to null (default) to avoid Terraform management of size." + type = number + default = null +} + +variable "max_size" { + description = "Maximum size of the HTCondor execute point pool." + type = number + default = 5 +} + +variable "min_idle" { + description = "Minimum number of idle VMs in the HTCondor pool (if pool reaches var.max_size, this minimum is not guaranteed); set to ensure jobs beginning run more quickly." + type = number + default = 0 +} + +variable "metadata" { + description = "Metadata to add to HTCondor execute points" + type = map(string) + default = {} +} + +# this default is deliberately the opposite of vm-instance because of observed +# issues running HTCondor docker universe jobs with OS Login enabled and running +# jobs as a user with uid>2^31; these uids occur when users outside the GCP +# organization login to a VM and OS Login is enabled. +variable "enable_oslogin" { + description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." + type = string + default = "ENABLE" + validation { + condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) + error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." + } +} + +variable "spot" { + description = "Provision VMs using discounted Spot pricing, allowing for preemption" + type = bool + default = false +} + +variable "disk_size_gb" { + description = "Boot disk size in GB" + type = number + default = 100 +} + +variable "disk_type" { + description = "Disk type for template" + type = string + default = "pd-balanced" +} + +variable "windows_startup_ps1" { + description = "Startup script to run at boot-time for Windows-based HTCondor execute points" + type = list(string) + default = [] + nullable = false +} + +variable "central_manager_ips" { + description = "List of IP addresses of HTCondor Central Managers" + type = list(string) +} + +variable "htcondor_bucket_name" { + description = "Name of HTCondor configuration bucket" + type = string +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = string, + count = number + })) + default = [] + nullable = false + + validation { + condition = length(var.guest_accelerator) <= 1 + error_message = "The HTCondor module supports 0 or 1 models of accelerator card on each execute point" + } +} + +variable "name_prefix" { + description = "Name prefix given to hostnames in this group of execute points; must be unique across all instances of this module" + type = string + nullable = false + validation { + condition = length(var.name_prefix) > 0 + error_message = "var.name_prefix must be a set to a non-empty string and must also be unique across all instances of htcondor-execute-point" + } +} + +variable "enable_shielded_vm" { + type = bool + default = false + description = "Enable the Shielded VM configuration (var.shielded_instance_config)." +} + +variable "shielded_instance_config" { + description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" + type = object({ + enable_secure_boot = bool + enable_vtpm = bool + enable_integrity_monitoring = bool + }) + + default = { + enable_secure_boot = true + enable_vtpm = true + enable_integrity_monitoring = true + } +} + +variable "update_policy" { + description = "Replacement policy for Access Point Managed Instance Group (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)" + type = string + default = "OPPORTUNISTIC" + validation { + condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) + error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf new file mode 100644 index 0000000000..47c905b352 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.1" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:htcondor-execute-point/v1.57.0" + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/README.md new file mode 100644 index 0000000000..278207b04a --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/README.md @@ -0,0 +1,45 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | > 5.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | > 5.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_instance_group_manager.mig](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_group_manager) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [base\_instance\_name](#input\_base\_instance\_name) | Base name for the instances in the MIG | `string` | `null` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment, will be used to name MIG if `var.name` is not provided | `string` | n/a | yes | +| [ghpc\_module\_id](#input\_ghpc\_module\_id) | Internal GHPC field, do not set this value | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to the MIG | `map(string)` | n/a | yes | +| [name](#input\_name) | Name of the MIG. If not provided, will be generated from `var.deployment_name` | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the MIG will be created | `string` | n/a | yes | +| [target\_size](#input\_target\_size) | Target number of instances in the MIG | `number` | `0` | no | +| [versions](#input\_versions) | Application versions managed by this instance group. Each version deals with a specific instance template | 
list(object({
    name              = string
    instance_template = string
    target_size = optional(object({
      fixed   = optional(number)
      percent = optional(number)
    }))
  }))
| n/a | yes | +| [wait\_for\_instances](#input\_wait\_for\_instances) | Whether to wait for all instances to be created/updated before returning | `bool` | `false` | no | +| [zone](#input\_zone) | Compute Platform zone. Required, currently only zonal MIGs are supported | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [self\_link](#output\_self\_link) | The URL of the created MIG | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/main.tf new file mode 100644 index 0000000000..0e7cf186c2 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/main.tf @@ -0,0 +1,85 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "mig", ghpc_role = "compute" }) +} + +locals { + sanitized_deploy_name = try(replace(lower(var.deployment_name), "/[^a-z0-9]/", ""), null) + sanitized_module_id = try(replace(lower(var.ghpc_module_id), "/[^a-z0-9]/", ""), null) + synth_mig_name = try("${local.sanitized_deploy_name}-${local.sanitized_module_id}", null) + + mig_name = var.name == null ? local.synth_mig_name : var.name + base_instance_name = var.base_instance_name == null ? local.mig_name : var.base_instance_name +} + +resource "google_compute_instance_group_manager" "mig" { + # REQUIRED + name = local.mig_name + base_instance_name = local.base_instance_name + zone = var.zone + + dynamic "version" { + for_each = var.versions + content { + name = version.value.name + instance_template = version.value.instance_template + dynamic "target_size" { + for_each = version.value.target_size != null ? [version.value.target_size] : [] + content { + fixed = target_size.value.fixed + percent = target_size.value.percent + } + } + } + } + + # OPTIONAL + project = var.project_id + target_size = var.target_size + wait_for_instances = var.wait_for_instances + + all_instances_config { + # TODO: validate that template metadata not getting wiped out + # TODO: validate that template labels not getting wiped out + labels = local.labels + } + + # OMITTED: + # * description + # * named_port + # * list_managed_instances_results + # * target_pools - specific for Load Balancers usage + # * wait_for_instances_status + # * auto_healing_policies + # * stateful_disk + # * stateful_internal_ip + # * update_policy + # * params + + + lifecycle { + precondition { + condition = local.mig_name != null + error_message = "Could not come up with a name for the MIG, specify `var.name`" + } + + precondition { + condition = local.base_instance_name != null + error_message = "Could not come up with a base_instance_name, specify `var.base_instance_name`" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/metadata.yaml new file mode 100644 index 0000000000..97a4fa9a89 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com +ghpc: + inject_module_id: ghpc_module_id diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/outputs.tf new file mode 100644 index 0000000000..23c66a3535 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/outputs.tf @@ -0,0 +1,18 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "self_link" { + description = "The URL of the created MIG" + value = google_compute_instance_group_manager.mig.self_link +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/variables.tf new file mode 100644 index 0000000000..b6c3c0e78a --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/variables.tf @@ -0,0 +1,86 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "Project in which the MIG will be created" + type = string +} + +variable "deployment_name" { + description = "Name of the deployment, will be used to name MIG if `var.name` is not provided" + type = string +} + +variable "labels" { + description = "Labels to add to the MIG" + type = map(string) +} + +variable "zone" { + description = "Compute Platform zone. Required, currently only zonal MIGs are supported" + type = string +} + + +variable "versions" { + description = <<-EOD + Application versions managed by this instance group. Each version deals with a specific instance template + EOD + type = list(object({ + name = string + instance_template = string + target_size = optional(object({ + fixed = optional(number) + percent = optional(number) + })) + })) + + validation { + condition = length(var.versions) > 0 + error_message = "At least one version must be provided" + } + +} + + +variable "ghpc_module_id" { + description = "Internal GHPC field, do not set this value" + type = string + default = null +} + +variable "name" { + description = "Name of the MIG. If not provided, will be generated from `var.deployment_name`" + type = string + default = null +} + +variable "base_instance_name" { + description = "Base name for the instances in the MIG" + type = string + default = null +} + + +variable "target_size" { + description = "Target number of instances in the MIG" + type = number + default = 0 +} + +variable "wait_for_instances" { + description = "Whether to wait for all instances to be created/updated before returning" + type = bool + default = false +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/versions.tf new file mode 100644 index 0000000000..a1045b9a6b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.3" + + required_providers { + google = { + source = "hashicorp/google" + version = "> 5.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:mig/v1.57.0" + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/README.md new file mode 100644 index 0000000000..506677ede8 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/README.md @@ -0,0 +1,112 @@ +# Description + +This module creates the Vertex AI Notebook, to be used in tutorials. + +Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +## Usage + +This is a simple usage, using the default network: + +```yaml + - id: bucket + source: community/modules/file-system/cloud-storage-bucket + settings: + name_prefix: my-bucket + local_mount: /home/jupyter/my-bucket + + - id: notebook + source: community/modules/compute/notebook + use: [bucket] + settings: + name_prefix: notebook + machine_type: n1-standard-4 + +``` + +If the user wants do specify a custom subnetwork, or specific external IP restrictions, they can use the `network_interfaces` variable, here is an example on how to use a Shared VPC Subnet with an ephemeral external IP: + +```yaml + - id: bucket + source: community/modules/file-system/cloud-storage-bucket + settings: + name_prefix: my-bucket + local_mount: /home/jupyter/my-bucket + + - id: notebook + source: community/modules/compute/notebook + use: [bucket] + settings: + name_prefix: notebook + machine_type: n1-standard-4 + network_interfaces: + - network: "projects/HOST_PROJECT_ID/global/networks/SHARED_VPC_NAME" + subnet: "projects/HOST_PROJECT_ID/regions/REGION/subnetworks/SUBNET_NAME" + nic_type: "VIRTIO_NET" +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [google](#requirement\_google) | >= 5.34 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 5.34 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.mount_script](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_workbench_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/workbench_instance) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment; used as part of name of the notebook. | `string` | n/a | yes | +| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | Bucket name, can be provided from the google-cloud-storage module | `string` | `null` | no | +| [instance\_image](#input\_instance\_image) | Instance Image | `map(string)` | 
{
  "family": "tf-latest-cpu",
  "name": null,
  "project": "deeplearning-platform-release"
}
| no | +| [labels](#input\_labels) | Labels to add to the resource Key-value pairs. | `map(string)` | n/a | yes | +| [machine\_type](#input\_machine\_type) | The machine type to employ | `string` | n/a | yes | +| [mount\_runner](#input\_mount\_runner) | mount content from the google-cloud-storage module | `map(string)` | n/a | yes | +| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces for the VM instance. Each network interface is represented by an object with the following fields:

- network: (Optional) The name of the Virtual Private Cloud (VPC) network that this VM instance is connected to.

- subnet: (Optional) The name of the subnetwork within the specified VPC that this VM instance is connected to.

- nic\_type: (Optional) The type of vNIC to be used on this interface. Possible values are: `VIRTIO_NET`, `GVNIC`.

- access\_configs: (Optional) An array of access configurations for this network interface. The access\_config object contains:
* external\_ip: (Required) An external IP address associated with this instance. Specify an unused static external IP address available to the project or leave this field undefined to use an IP from a shared ephemeral IP address pool. If you specify a static external IP address, it must live in the same region as the zone of the instance. | 
list(object({
    network  = optional(string)
    subnet   = optional(string)
    nic_type = optional(string)
    access_configs = optional(list(object({
      external_ip = optional(string)
    })))
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | ID of project in which the notebook will be created. | `string` | n/a | yes | +| [service\_account\_email](#input\_service\_account\_email) | If defined, the instance will use the service account specified instead of the Default Compute Engine Service Account | `string` | `null` | no | +| [zone](#input\_zone) | The zone to deploy to | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/main.tf new file mode 100644 index 0000000000..ad75526f50 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/main.tf @@ -0,0 +1,96 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "notebook", ghpc_role = "compute" }) +} + +locals { + suffix = random_id.resource_name_suffix.hex + #name = "thenotebook" + name = "notebook-${var.deployment_name}-${local.suffix}" + bucket = replace(var.gcs_bucket_path, "gs://", "") + post_script_filename = "mount-${local.suffix}.sh" + + # mount_runner_args is defined here: https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/3abddcfbd245b0e6747917a4e55b30658414ffd7/community/modules/file-system/cloud-storage-bucket/outputs.tf#L40 + mount_args = split(" ", var.mount_runner.args) + + unused = local.mount_args[0] + remote_mount = local.mount_args[1] + local_mount = local.mount_args[2] + fs_type = local.mount_args[3] + # These options provide a "rw" mount of the GCS bucket + mount_options = "defaults,_netdev,allow_other,implicit_dirs,gid=1000,uid=1000" + + content0 = var.mount_runner.content + content1 = replace(local.content0, "$1", local.unused) + content2 = replace(local.content1, "$2", local.remote_mount) + content3 = replace(local.content2, "$3", local.local_mount) + content4 = replace(local.content3, "$4", local.fs_type) + content5 = replace(local.content4, "$5", local.mount_options) + +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_storage_bucket_object" "mount_script" { + name = local.post_script_filename + content = local.content5 + bucket = local.bucket +} + +resource "google_workbench_instance" "instance" { + name = local.name + location = var.zone + project = var.project_id + labels = local.labels + gce_setup { + machine_type = var.machine_type + metadata = { + post-startup-script = "${var.gcs_bucket_path}/${google_storage_bucket_object.mount_script.name}" + } + vm_image { + project = var.instance_image.project + family = var.instance_image.family + } + + dynamic "service_accounts" { + for_each = var.service_account_email == null ? [] : [1] + content { + email = var.service_account_email + } + } + + dynamic "network_interfaces" { + for_each = var.network_interfaces + content { + network = network_interfaces.value.network + subnet = network_interfaces.value.subnet + nic_type = network_interfaces.value.nic_type + + dynamic "access_configs" { + for_each = network_interfaces.value.access_configs != null ? network_interfaces.value.access_configs : [] + content { + external_ip = access_configs.value.external_ip + } + } + } + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/metadata.yaml new file mode 100644 index 0000000000..4a7d5397ca --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - notebooks.googleapis.com + - storage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/variables.tf new file mode 100644 index 0000000000..4359de8c10 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/variables.tf @@ -0,0 +1,111 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which the notebook will be created." + type = string +} + +variable "deployment_name" { + description = "Name of the HPC deployment; used as part of name of the notebook." + type = string + # notebook name can have: lowercase letters, numbers, or hyphens (-) and cannot end with a hyphen + validation { + error_message = "The notebook name uses 'deployment_name' -- can only have: lowercase letters, numbers, or hyphens" + condition = can(regex("^[a-z0-9]+(?:-[a-z0-9]+)*$", var.deployment_name)) + } +} + +variable "zone" { + description = "The zone to deploy to" + type = string +} + +variable "machine_type" { + description = "The machine type to employ" + type = string +} + +variable "labels" { + description = "Labels to add to the resource Key-value pairs." + type = map(string) +} + +variable "instance_image" { + description = "Instance Image" + type = map(string) + default = { + project = "deeplearning-platform-release" + family = "tf-latest-cpu" + name = null + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "gcs_bucket_path" { + description = "Bucket name, can be provided from the google-cloud-storage module" + type = string + default = null +} + +variable "mount_runner" { + description = "mount content from the google-cloud-storage module" + type = map(string) + + validation { + condition = (length(split(" ", var.mount_runner.args)) == 5) + error_message = "There must be 5 elements in the Mount Runner Arguments: ${var.mount_runner.args} \n " + } +} + +variable "service_account_email" { + description = "If defined, the instance will use the service account specified instead of the Default Compute Engine Service Account" + type = string + default = null +} + +variable "network_interfaces" { + type = list(object({ + network = optional(string) + subnet = optional(string) + nic_type = optional(string) + access_configs = optional(list(object({ + external_ip = optional(string) + }))) + })) + default = [] + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 5.11 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 5.11 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | +| [slurm\_nodeset\_template](#module\_slurm\_nodeset\_template) | ../../internal/slurm-gcp/instance_template | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_compute_image.slurm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | +| [additional\_disks](#input\_additional\_disks) | Configurations of additional disks to be included on the partition nodes. | 
list(object({
    disk_name    = string
    device_name  = string
    disk_size_gb = number
    disk_type    = string
    disk_labels  = map(string)
    auto_delete  = bool
    boot         = bool
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | +| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | +| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of boot disk to create for the partition compute nodes. | `number` | `50` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-standard"` | no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | +| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [enable\_spot\_vm](#input\_enable\_spot\_vm) | Enable the partition to use spot VMs (https://cloud.google.com/spot-vms). | `bool` | `false` | no | +| [feature](#input\_feature) | The node feature, used to bind nodes to the nodeset. If not set, the nodeset name will be used. | `string` | `null` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm node group VM instances.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-9-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | +| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | +| [labels](#input\_labels) | Labels to add to partition compute instances. Key-value pairs. | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Compute Platform machine type to use for this partition compute nodes. | `string` | `"c2-standard-60"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | +| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy.

Note: Placement groups are not supported when on\_host\_maintenance is set to
"MIGRATE" and will be deactivated regardless of the value of
enable\_placement. To support enable\_placement, ensure on\_host\_maintenance is
set to "TERMINATE". | `string` | `"TERMINATE"` | no | +| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources. | `string` | n/a | yes | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the compute instances. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the compute instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | Path to the Slurm bucket. | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster. | `string` | n/a | yes | +| [spot\_instance\_config](#input\_spot\_instance\_config) | Configuration for spot VMs. | 
object({
    termination_action = string
  })
| `null` | no | +| [startup\_script](#input\_startup\_script) | Startup script used by VMs in this nodeset | `string` | `"# no-op"` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | +| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instance\_template\_self\_link](#output\_instance\_template\_self\_link) | The URI of the template. | +| [node\_name\_prefix](#output\_node\_name\_prefix) | The prefix to be used for the node names.

Make sure that nodes are named `-`
This temporary required for proper functioning of the nodes.
While Slurm scheduler uses "features" to bind node and nodeset,
the SlurmGCP relies on node names for this (to be switched to features as well). | +| [nodeset\_dyn](#output\_nodeset\_dyn) | Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf new file mode 100644 index 0000000000..e52ddb972c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf @@ -0,0 +1,121 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset-dynamic", ghpc_role = "compute" }) +} + +module "gpu" { + source = "../../../../modules/internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + nodeset_name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) + feature = coalesce(var.feature, local.nodeset_name) + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + universe_domain = { "universe_domain" = var.universe_domain } + + metadata = merge( + local.disable_automatic_updates_metadata, + local.universe_domain, + { slurmd_feature = local.feature }, + var.metadata + ) + + nodeset = { + nodeset_name = local.nodeset_name + nodeset_feature : local.feature + startup_script = local.ghpc_startup_script + network_storage = var.network_storage + } + + additional_disks = [ + for ad in var.additional_disks : { + disk_name = ad.disk_name + device_name = ad.device_name + disk_type = ad.disk_type + disk_size_gb = ad.disk_size_gb + disk_labels = merge(ad.disk_labels, local.labels) + auto_delete = ad.auto_delete + boot = ad.boot + } + ] + + public_access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] + access_config = length(var.access_config) == 0 ? local.public_access_config : var.access_config + + service_account = { + email = var.service_account_email + scopes = var.service_account_scopes + } + + ghpc_startup_script = [{ + filename = "ghpc_nodeset_startup.sh" + content = var.startup_script + }] + +} + +module "slurm_nodeset_template" { + source = "../../internal/slurm-gcp/instance_template" + + project_id = var.project_id + region = var.region + name_prefix = local.nodeset_name + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = "compute" + slurm_bucket_path = var.slurm_bucket_path + metadata = local.metadata + + additional_disks = local.additional_disks + disk_auto_delete = var.disk_auto_delete + disk_labels = merge(local.labels, var.disk_labels) + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + + bandwidth_tier = var.bandwidth_tier + can_ip_forward = var.can_ip_forward + + advanced_machine_features = var.advanced_machine_features + enable_confidential_vm = var.enable_confidential_vm + enable_oslogin = var.enable_oslogin + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config + + labels = local.labels + machine_type = var.machine_type + + min_cpu_platform = var.min_cpu_platform + on_host_maintenance = var.on_host_maintenance + termination_action = try(var.spot_instance_config.termination_action, null) + preemptible = var.preemptible + spot = var.enable_spot_vm + service_account = local.service_account + gpu = one(local.guest_accelerator) # requires gpu_definition.tf + source_image_family = local.source_image_family # requires source_image_logic.tf + source_image_project = local.source_image_project_normalized # requires source_image_logic.tf + source_image = local.source_image # requires source_image_logic.tf + + subnetwork = var.subnetwork_self_link + additional_networks = var.additional_networks + access_config = local.access_config + tags = concat([var.slurm_cluster_name], var.tags) +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml new file mode 100644 index 0000000000..a99e59d09f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [compute.googleapis.com] +ghpc: + inject_module_id: name diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf new file mode 100644 index 0000000000..2d2d1415cf --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf @@ -0,0 +1,36 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "nodeset_dyn" { + description = "Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`." + value = local.nodeset +} + +output "instance_template_self_link" { + description = "The URI of the template." + value = module.slurm_nodeset_template.self_link +} + +output "node_name_prefix" { + description = <<-EOD + The prefix to be used for the node names. + + Make sure that nodes are named `-` + This temporary required for proper functioning of the nodes. + While Slurm scheduler uses "features" to bind node and nodeset, + the SlurmGCP relies on node names for this (to be switched to features as well). + EOD + value = "${var.slurm_cluster_name}-${local.nodeset_name}" + +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf new file mode 100644 index 0000000000..0641cea7a8 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf @@ -0,0 +1,77 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # Currently supported images and projects + known_project_families = { + schedmd-slurm-public = [ + "slurm-gcp-6-9-debian-11", + "slurm-gcp-6-9-hpc-rocky-linux-8", + "slurm-gcp-6-9-ubuntu-2004-lts", + "slurm-gcp-6-9-ubuntu-2204-lts-arm64" + ] + } + + # This approach to "hacking" the project name allows a chain of Terraform + # calls to set the instance source_image (boot disk) with a "relative + # resource name" that passes muster with VPC Service Control rules + # + # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 + # https://cloud.google.com/apis/design/resource_names#relative_resource_name + source_image_project_normalized = (can(var.instance_image.family) ? + "projects/${data.google_compute_image.slurm.project}/global/images/family" : + "projects/${data.google_compute_image.slurm.project}/global/images" + ) + source_image_family = can(var.instance_image.family) ? data.google_compute_image.slurm.family : "" + source_image = can(var.instance_image.name) ? data.google_compute_image.slurm.name : "" +} + +data "google_compute_image" "slurm" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = var.instance_image.project + + lifecycle { + precondition { + condition = length(regexall("^projects/.+?/global/images/family$", var.instance_image.project)) == 0 + error_message = "The \"project\" field in var.instance_image no longer supports a long-form ending in \"family\". Specify only the project ID." + } + + postcondition { + condition = var.instance_image_custom || contains(keys(local.known_project_families), self.project) + error_message = <<-EOD + Images in project ${self.project} are not published by SchedMD. Images must be created by compatible releases of the Terraform and Packer modules following the guidance at https://goo.gle/hpc-slurm-images. Set var.instance_image_custom to true to silence this error and acknowledge that you are using a compatible image. + EOD + } + postcondition { + condition = !contains(keys(local.known_project_families), self.project) || try(contains(local.known_project_families[self.project], self.family), false) + error_message = <<-EOD + Image family ${self.family} published by SchedMD in project ${self.project} is not compatible with this release of the Terraform Slurm modules. Select from known compatible releases: + ${join("\n", [for p in try(local.known_project_families[self.project], []) : "\t\"${p}\""])} + EOD + } + postcondition { + condition = var.disk_size_gb >= self.disk_size_gb + error_message = "'disk_size_gb: ${var.disk_size_gb}' is smaller than the image size (${self.disk_size_gb}GB), please increase the blueprint disk size" + } + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf new file mode 100644 index 0000000000..595e424b81 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf @@ -0,0 +1,402 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "name" { + description = <<-EOD + Name of the nodeset. Automatically populated by the module id if not set. + If setting manually, ensure a unique value across all nodesets. + EOD + type = string +} + +variable "feature" { + type = string + description = "The node feature, used to bind nodes to the nodeset. If not set, the nodeset name will be used." + default = null +} + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "slurm_cluster_name" { + description = "Name of the Slurm cluster." + type = string +} + +variable "slurm_bucket_path" { + description = "Path to the Slurm bucket." + type = string +} + + +variable "machine_type" { + description = "Compute Platform machine type to use for this partition compute nodes." + type = string + default = "c2-standard-60" +} + +variable "metadata" { + type = map(string) + description = "Metadata, provided as a map." + default = {} +} + +variable "instance_image" { + description = <<-EOD + Defines the image that will be used in the Slurm node group VM instances. + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + + For more information on creating custom images that comply with Slurm on GCP + see the "Slurm on GCP Custom Images" section in docs/vm-images.md. + EOD + type = map(string) + default = { + family = "slurm-gcp-6-9-hpc-rocky-linux-8" + project = "schedmd-slurm-public" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "instance_image_custom" { + description = <<-EOD + A flag that designates that the user is aware that they are requesting + to use a custom and potentially incompatible image for this Slurm on + GCP module. + + If the field is set to false, only the compatible families and project + names will be accepted. The deployment will fail with any other image + family or name. If set to true, no checks will be done. + + See: https://goo.gle/hpc-slurm-images + EOD + type = bool + default = false +} + + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "tags" { + type = list(string) + description = "Network tag list." + default = [] +} + +variable "disk_type" { + description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." + type = string + default = "pd-standard" +} + +variable "disk_size_gb" { + description = "Size of boot disk to create for the partition compute nodes." + type = number + default = 50 +} + +variable "disk_auto_delete" { + type = bool + description = "Whether or not the boot disk should be auto-deleted." + default = true +} + +variable "disk_labels" { + description = "Labels specific to the boot disk. These will be merged with var.labels." + type = map(string) + default = {} +} + +variable "additional_disks" { + description = "Configurations of additional disks to be included on the partition nodes." + type = list(object({ + disk_name = string + device_name = string + disk_size_gb = number + disk_type = string + disk_labels = map(string) + auto_delete = bool + boot = bool + })) + default = [] +} + +variable "enable_confidential_vm" { + type = bool + description = "Enable the Confidential VM configuration. Note: the instance image must support option." + default = false +} + +variable "enable_shielded_vm" { + type = bool + description = "Enable the Shielded VM configuration. Note: the instance image must support option." + default = false +} + +variable "shielded_instance_config" { + type = object({ + enable_integrity_monitoring = bool + enable_secure_boot = bool + enable_vtpm = bool + }) + description = <<-EOD + Shielded VM configuration for the instance. Note: not used unless + enable_shielded_vm is 'true'. + - enable_integrity_monitoring : Compare the most recent boot measurements to the + integrity policy baseline and return a pair of pass/fail results depending on + whether they match or not. + - enable_secure_boot : Verify the digital signature of all boot components, and + halt the boot process if signature verification fails. + - enable_vtpm : Use a virtualized trusted platform module, which is a + specialized computer chip you can use to encrypt objects like keys and + certificates. + EOD + default = { + enable_integrity_monitoring = true + enable_secure_boot = true + enable_vtpm = true + } +} + + +variable "enable_oslogin" { + type = bool + description = <<-EOD + Enables Google Cloud os-login for user login and authentication for VMs. + See https://cloud.google.com/compute/docs/oslogin + EOD + default = true +} + +variable "can_ip_forward" { + description = "Enable IP forwarding, for NAT instances for example." + type = bool + default = false +} + +variable "advanced_machine_features" { + description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" + type = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) + default = { + threads_per_core = 1 # disable SMT by default + } +} + +variable "enable_smt" { # tflint-ignore: terraform_unused_declarations + type = bool + description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + default = null + validation { + condition = var.enable_smt == null + error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + } +} + +variable "labels" { + description = "Labels to add to partition compute instances. Key-value pairs." + type = map(string) + default = {} +} + +variable "min_cpu_platform" { + description = "The name of the minimum CPU platform that you want the instance to use." + type = string + default = null +} + +variable "on_host_maintenance" { + type = string + description = <<-EOD + Instance availability Policy. + + Note: Placement groups are not supported when on_host_maintenance is set to + "MIGRATE" and will be deactivated regardless of the value of + enable_placement. To support enable_placement, ensure on_host_maintenance is + set to "TERMINATE". + EOD + default = "TERMINATE" +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = string, + count = number + })) + default = [] + nullable = false + + validation { + condition = length(var.guest_accelerator) <= 1 + error_message = "The Slurm modules supports 0 or 1 models of accelerator card on each node." + } +} + +variable "preemptible" { + description = "Should use preemptibles to burst." + type = bool + default = false +} + + +variable "service_account_email" { + description = "Service account e-mail address to attach to the compute instances." + type = string + default = null +} + +variable "service_account_scopes" { + description = "Scopes to attach to the compute instances." + type = set(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +variable "enable_spot_vm" { + description = "Enable the partition to use spot VMs (https://cloud.google.com/spot-vms)." + type = bool + default = false +} + +variable "spot_instance_config" { + description = "Configuration for spot VMs." + type = object({ + termination_action = string + }) + default = null +} + +variable "bandwidth_tier" { + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [accelerator\_config](#input\_accelerator\_config) | Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details. | 
object({
    topology = string
    version  = string
  })
| 
{
  "topology": "",
  "version": ""
}
| no | +| [data\_disks](#input\_data\_disks) | The data disks to include in the TPU node | `list(string)` | `[]` | no | +| [disable\_public\_ips](#input\_disable\_public\_ips) | DEPRECATED: Use `enable_public_ips` instead. | `bool` | `null` | no | +| [docker\_image](#input\_docker\_image) | The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf- | `string` | `null` | no | +| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | +| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | +| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of auto-scaling worker nodes allowed in this partition.
For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores).
See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. | `number` | `0` | no | +| [node\_count\_static](#input\_node\_count\_static) | Number of worker nodes to be statically created.
For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores).
See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. | `number` | `0` | no | +| [node\_type](#input\_node\_type) | Specify a node type to base the vm configuration upon it. | `string` | `""` | no | +| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | +| [preserve\_tpu](#input\_preserve\_tpu) | Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [reserved](#input\_reserved) | Specify whether TPU-vms in this nodeset are created under a reservation. | `bool` | `false` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the TPU-vm. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the TPU-vm. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The name of the subnetwork to attach the TPU-vm of this nodeset to. | `string` | n/a | yes | +| [tf\_version](#input\_tf\_version) | Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details. | `string` | `"2.14.0"` | no | +| [zone](#input\_zone) | Zone in which to create compute VMs. TPU partitions can only specify a single zone. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [nodeset\_tpu](#output\_nodeset\_tpu) | Details of the nodeset tpu. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf new file mode 100644 index 0000000000..ac9b119702 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf @@ -0,0 +1,59 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# locals { +# # This label allows for billing report tracking based on module. +# labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset", ghpc_role = "compute" }) +# } + +locals { + name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) + + service_account = { + email = var.service_account_email + scopes = var.service_account_scopes + } + + nodeset_tpu = { + node_count_static = var.node_count_static + node_count_dynamic_max = var.node_count_dynamic_max + nodeset_name = local.name + node_type = var.node_type + + accelerator_config = var.accelerator_config + tf_version = var.tf_version + preemptible = var.preemptible + preserve_tpu = var.preserve_tpu + + data_disks = var.data_disks + docker_image = var.docker_image + + enable_public_ip = var.enable_public_ips + # TODO: rename to subnetwork_self_link, requires changes to the scripts + subnetwork = var.subnetwork_self_link + service_account = local.service_account + zone = var.zone + + project_id = var.project_id + reserved = var.reserved + network_storage = var.network_storage + } + + node_type_core_count = var.node_type == "" ? 0 : tonumber(regex("-(.*)", var.node_type)[0]) + + accelerator_core_list = var.accelerator_config.topology == "" ? [0, 0] : regexall("\\d+", var.accelerator_config.topology) + accelerator_core_count = length(local.accelerator_core_list) > 2 ? (local.accelerator_core_list[0] * local.accelerator_core_list[1] * local.accelerator_core_list[2]) * 2 : (local.accelerator_core_list[0] * local.accelerator_core_list[1]) * 2 + + tpu_core_count = local.accelerator_core_count == 0 ? local.node_type_core_count : local.accelerator_core_count +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml new file mode 100644 index 0000000000..95b6d1c730 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] +ghpc: + inject_module_id: name + has_to_be_used: true diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf new file mode 100644 index 0000000000..8cb7b8663e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf @@ -0,0 +1,39 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "nodeset_tpu" { + description = "Details of the nodeset tpu. Typically used as input to `schedmd-slurm-gcp-v6-partition`." + value = local.nodeset_tpu + + precondition { + condition = (var.node_type == "") != (var.accelerator_config == { topology : "", version : "" }) + error_message = "Either a node_type or an accelerator_config must be provided." + } + + precondition { + condition = ((local.tpu_core_count / 8) <= var.node_count_dynamic_max) || ((local.tpu_core_count / 8) <= var.node_count_static) + error_message = <<-EOD + When using TPUs there should be at least one node per every 8 cores. + Currently there are ${local.tpu_core_count} cores but only ${var.node_count_static} static nodes and ${var.node_count_dynamic_max} dynamic nodes. + EOD + } + + precondition { + condition = (var.node_count_dynamic_max % (local.tpu_core_count / 8) == 0) && (var.node_count_static % (local.tpu_core_count / 8) == 0) + error_message = <<-EOD + The number of worker nodes should be a multiple of ${local.tpu_core_count / 8}. + This is to ensure each node has a TPU machine for job scheduling. + EOD + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf new file mode 100644 index 0000000000..367b0bee09 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf @@ -0,0 +1,171 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "node_count_static" { + description = <<-EOD + Number of worker nodes to be statically created. + For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores). + See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. + EOD + type = number + default = 0 +} + +variable "node_count_dynamic_max" { + description = <<-EOD + Maximum number of auto-scaling worker nodes allowed in this partition. + For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores). + See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. + EOD + type = number + default = 0 +} + +variable "name" { + description = <<-EOD + Name of the nodeset. Automatically populated by the module id if not set. + If setting manually, ensure a unique value across all nodesets. + EOD + type = string +} + +variable "enable_public_ips" { + description = "If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access_config is set." + type = bool + default = false +} + +variable "disable_public_ips" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: Use `enable_public_ips` instead." + type = bool + default = null + validation { + condition = var.disable_public_ips == null + error_message = "DEPRECATED: Use `enable_public_ips` instead." + } +} + +variable "node_type" { + description = "Specify a node type to base the vm configuration upon it." + type = string + default = "" +} + +variable "accelerator_config" { + description = "Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details." + type = object({ + topology = string + version = string + }) + default = { + topology = "" + version = "" + } + validation { + condition = var.accelerator_config.version == "" ? true : contains(["V2", "V3", "V4"], var.accelerator_config.version) + error_message = "accelerator_config.version must be one of [\"V2\", \"V3\", \"V4\"]" + } + validation { + condition = var.accelerator_config.topology == "" ? true : can(regex("^[1-9]x[1-9](x[1-9])?$", var.accelerator_config.topology)) + error_message = "accelerator_config.topology must be a valid topology, like 2x2 4x4x4 4x2x4 etc..." + } +} + +variable "tf_version" { + description = "Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details." + type = string + default = "2.14.0" +} + +variable "preemptible" { + description = "Should use preemptibles to burst." + type = bool + default = false +} + +variable "preserve_tpu" { + description = "Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted" + type = bool + default = false +} + +variable "zone" { + description = "Zone in which to create compute VMs. TPU partitions can only specify a single zone." + type = string +} + +variable "data_disks" { + description = "The data disks to include in the TPU node" + type = list(string) + default = [] +} + +variable "docker_image" { + description = "The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf-" + type = string + default = null +} + +variable "subnetwork_self_link" { + type = string + description = "The name of the subnetwork to attach the TPU-vm of this nodeset to." +} + +variable "service_account_email" { + description = "Service account e-mail address to attach to the TPU-vm." + type = string + default = null +} + +variable "service_account_scopes" { + description = "Scopes to attach to the TPU-vm." + type = set(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +variable "service_account" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." + type = object({ + email = string + scopes = set(string) + }) + default = null + validation { + condition = var.service_account == null + error_message = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." + } +} + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "reserved" { + description = "Specify whether TPU-vms in this nodeset are created under a reservation." + type = bool + default = false +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured on nodes." + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + })) + default = [] +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf new file mode 100644 index 0000000000..8b9d422d23 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.3" + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-tpu/v1.57.0" + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md new file mode 100644 index 0000000000..bb742e72d1 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md @@ -0,0 +1,226 @@ +## Description + +This module creates a nodeset data structure intended to be input to the +[schedmd-slurm-gcp-v6-partition](../schedmd-slurm-gcp-v6-partition/) module. + +Nodesets allow adding heterogeneous node types to a partition, and hence +running jobs that mix multiple node characteristics. See the [heterogeneous jobs +section][hetjobs] of the SchedMD documentation for more information. + +To specify nodes from a specific nodesets in a partition, the [`--nodelist`] +(or `-w`) flag can be used, for example: + +```bash +srun -N 3 -p compute --nodelist cluster-compute-group-[0-2] hostname +``` + +Where the 3 nodes will be selected from the nodes `cluster-compute-group-[0-2]` +in the compute partition. + +Additionally, depending on how the nodes differ, a constraint can be added via +the [`--constraint`] (or `-C`) flag or other flags such as `--mincpus` can be +used to specify nodes with the desired characteristics. + +[`--nodelist`]: https://slurm.schedmd.com/srun.html#OPT_nodelist +[`--constraint`]: https://slurm.schedmd.com/srun.html#OPT_constraint +[hetjobs]: https://slurm.schedmd.com/heterogeneous_jobs.html + +### Example + +The following code snippet creates a partition module using the `nodeset` +module as input with: + +* a max node count of 200 +* VM machine type of `c2-standard-30` +* partition name of "compute" +* default nodeset name of "ghpc" +* connected to the `network` module via `use` +* nodes mounted to homefs via `use` + +```yaml +- id: nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: + - network + settings: + node_count_dynamic_max: 200 + machine_type: c2-standard-30 + +- id: compute_partition + source: community/modules/compute/schedmd-slurm-gcp-v6-partition + use: + - homefs + - nodeset + settings: + partition_name: compute +``` + +## Custom Images + +For more information on creating valid custom images for the node group VM +instances or for custom instance templates, see our [vm-images.md] documentation +page. + +[vm-images.md]: ../../../../docs/vm-images.md#slurm-on-gcp-custom-images + +## GPU Support + +More information on GPU support in Slurm on GCP and other Cluster Toolkit modules +can be found at [docs/gpu-support.md](../../../../docs/gpu-support.md) + +### Compute VM Zone Policies + +The Slurm on GCP nodeset module allows you to specify additional zones in +which to create VMs through [bulk creation][bulk]. This is valuable when +configuring partitions with popular VM families and you desire access to +more compute resources across zones. + +[bulk]: https://cloud.google.com/compute/docs/instances/multiple/about-bulk-creation +[networkpricing]: https://cloud.google.com/vpc/network-pricing + +> **_WARNING:_** Lenient zone policies can lead to additional egress costs when +> moving large amounts of data between zones in the same region. For example, +> traffic between VMs and traffic from VMs to shared filesystems such as +> Filestore. For more information on egress fees, see the +> [Network Pricing][networkpricing] Google Cloud documentation. +> +> To avoid egress charges, ensure your compute nodes are created in a single +> zone by setting var.zone and leaving var.zones to its default value of the +> empty list. +> +> **_NOTE:_** If a new zone is added to the region while the cluster is active, +> nodes in the partition may be created in that zone. In this case, the +> partition may need to be redeployed to ensure the newly added zone is denied. + +In the zonal example below, the nodeset's zone implicitly defaults to the +deployment variable `vars.zone`: + +```yaml +vars: + zone: us-central1-f + +- id: zonal-nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset +``` + +In the example below, we enable creation in additional zones: + +```yaml +vars: + zone: us-central1-f + +- id: multi-zonal-nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + settings: + zones: + - us-central1-a + - us-central1-b +``` + +## Support +The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform +modules. For support with the underlying modules, see the instructions in the +[slurm-gcp README][slurm-gcp-readme]. + +[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp +[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4 | +| [google](#requirement\_google) | >= 5.11 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 5.11 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.machine_type_zone_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [google_compute_image.slurm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | +| [google_compute_machine_types.machine_types_by_zone](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_machine_types) | data source | +| [google_compute_reservation.reservation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_reservation) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | +| [additional\_disks](#input\_additional\_disks) | Configurations of additional disks to be included on the partition nodes. | 
list(object({
    disk_name                  = optional(string)
    device_name                = optional(string)
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = optional(map(string))
    auto_delete                = optional(bool)
    boot                       = optional(bool)
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = optional(string)
    subnetwork         = string
    subnetwork_project = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    stack_type         = optional(string)
    queue_count        = optional(number)
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | +| [disable\_public\_ips](#input\_disable\_public\_ips) | DEPRECATED: Use `enable_public_ips` instead. | `bool` | `null` | no | +| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | +| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | +| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of boot disk to create for the partition compute nodes. | `number` | `50` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-standard"` | no | +| [dws\_flex](#input\_dws\_flex) | If set and `enabled = true`, will utilize the DWS Flex Start to provision nodes.
See: https://cloud.google.com/blog/products/compute/introducing-dynamic-workload-scheduler
Options:
- enable: Enable DWS Flex Start
- max\_run\_duration: Maximum duration in seconds for the job to run, should not exceed 604,800 (one week).
- use\_job\_duration: Use the job duration to determine the max\_run\_duration, if job duration is not set, max\_run\_duration will be used.
- use\_bulk\_insert: Uses the legacy implementation of DWS Flex Start with Bulk Insert for non-accelerator instances

Limitations:
- CAN NOT be used with reservations;
- CAN NOT be used with placement groups;
- If `use_job_duration` is enabled nodeset can be used in "exclusive" partitions only | 
object({
    enabled          = optional(bool, true)
    max_run_duration = optional(number, 604800) # one week
    use_job_duration = optional(bool, false)
    use_bulk_insert  = optional(bool, false)
  })
| 
{
  "enabled": false
}
| no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_maintenance\_reservation](#input\_enable\_maintenance\_reservation) | Enables slurm reservation for scheduled maintenance. | `bool` | `false` | no | +| [enable\_opportunistic\_maintenance](#input\_enable\_opportunistic\_maintenance) | On receiving maintenance notification, maintenance will be performed as soon as nodes becomes idle. | `bool` | `false` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | +| [enable\_placement](#input\_enable\_placement) | Use placement policy for VMs in this nodeset.
See: https://cloud.google.com/compute/docs/instances/placement-policies-overview
To set max\_distance of used policy, use `placement_max_distance` variable.

Enabled by default, reasons for users to disable it:
- If non-dense reservation is used, user can avoid extra-cost of creating placement policies;
- If user wants to avoid "all or nothing" VM provisioning behaviour;
- If user wants to intentionally have "spread" VMs (e.g. for reliability reasons) | `bool` | `true` | no | +| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [enable\_spot\_vm](#input\_enable\_spot\_vm) | Enable the partition to use spot VMs (https://cloud.google.com/spot-vms). | `bool` | `false` | no | +| [future\_reservation](#input\_future\_reservation) | If set, will make use of the future reservation for the nodeset. Input can be either the future reservation name or its selfLink in the format 'projects/PROJECT\_ID/zones/ZONE/futureReservations/FUTURE\_RESERVATION\_NAME'.
See https://cloud.google.com/compute/docs/instances/future-reservations-overview | `string` | `""` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm node group VM instances.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-9-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | +| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | +| [instance\_properties](#input\_instance\_properties) | Override the instance properties. Used to test features not supported by Slurm GCP,
recommended for advanced usage only.
See https://cloud.google.com/compute/docs/reference/rest/v1/regionInstances/bulkInsert
If any sub-field (e.g. scheduling) is set, it will override the values computed by
SlurmGCP and ignoring values of provided vars. | `any` | `null` | no | +| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for compute nodes. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to partition compute instances. Key-value pairs. | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Compute Platform machine type to use for this partition compute nodes. | `string` | `"c2-standard-60"` | no | +| [maintenance\_interval](#input\_maintenance\_interval) | Sets the maintenance interval for instances in this nodeset.
See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#maintenance_interval. | `string` | `null` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | +| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | +| [node\_conf](#input\_node\_conf) | Map of Slurm node line configuration. | `map(any)` | `{}` | no | +| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of auto-scaling nodes allowed in this partition. | `number` | `10` | no | +| [node\_count\_static](#input\_node\_count\_static) | Number of nodes to be statically created. | `number` | `0` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy.

Note: Placement groups are not supported when on\_host\_maintenance is set to
"MIGRATE" and will be deactivated regardless of the value of
enable\_placement. To support enable\_placement, ensure on\_host\_maintenance is
set to "TERMINATE". | `string` | `"TERMINATE"` | no | +| [placement\_max\_distance](#input\_placement\_max\_distance) | Maximum distance between nodes in the placement group. Requires enable\_placement to be true. Values must be supported by the chosen machine type. | `number` | `null` | no | +| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources. | `string` | n/a | yes | +| [reservation\_name](#input\_reservation\_name) | Name of the reservation to use for VM resources, should be in one of the following formats:
- projects/PROJECT\_ID/reservations/RESERVATION\_NAME[/reservationBlocks/BLOCK\_ID]
- RESERVATION\_NAME[/reservationBlocks/BLOCK\_ID]

Must be a "SPECIFIC" reservation
Set to empty string if using no reservation or automatically-consumed reservations | `string` | `""` | no | +| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the compute instances. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the compute instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [spot\_instance\_config](#input\_spot\_instance\_config) | Configuration for spot VMs. | 
object({
    termination_action = string
  })
| `null` | no | +| [startup\_script](#input\_startup\_script) | Startup script used by VMs in this nodeset | `string` | `"# no-op"` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | +| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | +| [zone](#input\_zone) | Zone in which to create compute VMs. Additional zones in the same region can be specified in var.zones. | `string` | n/a | yes | +| [zone\_target\_shape](#input\_zone\_target\_shape) | Strategy for distributing VMs across zones in a region.
ANY
GCE picks zones for creating VM instances to fulfill the requested number of VMs
within present resource constraints and to maximize utilization of unused zonal
reservations.
ANY\_SINGLE\_ZONE (default)
GCE always selects a single zone for all the VMs, optimizing for resource quotas,
available reservations and general capacity.
BALANCED
GCE prioritizes acquisition of resources, scheduling VMs in zones where resources
are available while distributing VMs as evenly as possible across allowed zones
to minimize the impact of zonal failure. | `string` | `"ANY_SINGLE_ZONE"` | no | +| [zones](#input\_zones) | Additional zones in which to allow creation of partition nodes. Google Cloud
will find zone based on availability, quota and reservations.
Should not be set if SPECIFIC reservation is used. | `set(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [nodeset](#output\_nodeset) | Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf new file mode 100644 index 0000000000..5cb0e9751f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf @@ -0,0 +1,224 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset", ghpc_role = "compute" }) +} + +module "gpu" { + source = "../../../../modules/internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + metadata = merge( + local.disable_automatic_updates_metadata, + var.metadata + ) + + name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) + + additional_disks = [ + for ad in var.additional_disks : { + disk_name = ad.disk_name + device_name = ad.device_name + disk_type = ad.disk_type + disk_size_gb = ad.disk_size_gb + disk_labels = merge(ad.disk_labels, local.labels) + auto_delete = ad.auto_delete + boot = ad.boot + disk_resource_manager_tags = ad.disk_resource_manager_tags + } + ] + + public_access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] + access_config = length(var.access_config) == 0 ? local.public_access_config : var.access_config + + service_account = { + email = var.service_account_email + scopes = var.service_account_scopes + } + + ghpc_startup_script = [{ + filename = "ghpc_nodeset_startup.sh" + content = var.startup_script + }] + + termination_action = (var.dws_flex.enabled && !var.dws_flex.use_bulk_insert) ? "DELETE" : try(var.spot_instance_config.termination_action, null) + + nodeset = { + node_count_static = var.node_count_static + node_count_dynamic_max = var.node_count_dynamic_max + node_conf = var.node_conf + nodeset_name = local.name + dws_flex = var.dws_flex + + disk_auto_delete = var.disk_auto_delete + disk_labels = merge(local.labels, var.disk_labels) + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + disk_resource_manager_tags = var.disk_resource_manager_tags + additional_disks = local.additional_disks + + bandwidth_tier = var.bandwidth_tier + can_ip_forward = var.can_ip_forward + + enable_confidential_vm = var.enable_confidential_vm + enable_placement = var.enable_placement + placement_max_distance = var.placement_max_distance + enable_oslogin = var.enable_oslogin + enable_shielded_vm = var.enable_shielded_vm + gpu = one(local.guest_accelerator) + + labels = local.labels + machine_type = terraform_data.machine_type_zone_validation.output + advanced_machine_features = var.advanced_machine_features + metadata = local.metadata + min_cpu_platform = var.min_cpu_platform + + on_host_maintenance = var.on_host_maintenance + preemptible = var.preemptible + region = var.region + resource_manager_tags = var.resource_manager_tags + service_account = local.service_account + shielded_instance_config = var.shielded_instance_config + source_image_family = local.source_image_family # requires source_image_logic.tf + source_image_project = local.source_image_project_normalized # requires source_image_logic.tf + source_image = local.source_image # requires source_image_logic.tf + subnetwork_self_link = var.subnetwork_self_link + additional_networks = var.additional_networks + access_config = local.access_config + tags = var.tags + spot = var.enable_spot_vm + termination_action = local.termination_action + reservation_name = local.reservation_name + future_reservation = local.future_reservation + maintenance_interval = var.maintenance_interval + instance_properties_json = jsonencode(var.instance_properties) + + zone_target_shape = var.zone_target_shape + zone_policy_allow = local.zones + zone_policy_deny = local.zones_deny + + startup_script = local.ghpc_startup_script + network_storage = var.network_storage + + enable_maintenance_reservation = var.enable_maintenance_reservation + enable_opportunistic_maintenance = var.enable_opportunistic_maintenance + } +} + +locals { + zones = setunion(var.zones, [var.zone]) + zones_deny = setsubtract(data.google_compute_zones.available.names, local.zones) +} + +data "google_compute_zones" "available" { + project = var.project_id + region = var.region + + lifecycle { + postcondition { + condition = length(setsubtract(local.zones, self.names)) == 0 + error_message = <<-EOD + Invalid zones=${jsonencode(setsubtract(local.zones, self.names))} + Available zones=${jsonencode(self.names)} + EOD + } + } +} + +locals { + res_match = regex("^(?P(?Pprojects/(?P[a-z0-9-]+)/reservations/)?(?P[a-z0-9-]+)(?P/reservationBlocks/[a-z0-9-]+)?)?$", var.reservation_name) + + res_short_name = local.res_match.name + res_project = coalesce(local.res_match.project, var.project_id) + res_prefix = coalesce(local.res_match.prefix, "projects/${local.res_project}/reservations/") + res_suffix = local.res_match.suffix == null ? "" : local.res_match.suffix + + reservation_name = local.res_match.whole == null ? "" : "${local.res_prefix}${local.res_short_name}${local.res_suffix}" +} + +locals { + fr_match = regex("^(?Pprojects/(?P[a-z0-9-]+)/zones/(?P[a-z0-9-]+)/futureReservations/)?(?P[a-z0-9-]+)?$", var.future_reservation) + + fr_name = local.fr_match.name + fr_project = coalesce(local.fr_match.project, var.project_id) + fr_zone = coalesce(local.fr_match.zone, var.zone) + + future_reservation = var.future_reservation == "" ? "" : "projects/${local.fr_project}/zones/${local.fr_zone}/futureReservations/${local.fr_name}" +} + + +# tflint-ignore: terraform_unused_declarations +data "google_compute_reservation" "reservation" { + count = length(local.reservation_name) > 0 ? 1 : 0 + + name = local.res_short_name + project = local.res_project + zone = var.zone + + lifecycle { + postcondition { + condition = self.self_link != null + error_message = "Couldn't find the reservation ${var.reservation_name}" + } + + postcondition { + condition = coalesce(self.specific_reservation_required, true) + error_message = < 0] +} + +resource "terraform_data" "machine_type_zone_validation" { + input = var.machine_type + lifecycle { + precondition { + condition = length(local.zones_with_machine_type) > 0 + error_message = <<-EOT + machine type ${var.machine_type} is not available in any of the zones ${jsonencode(local.zones)}". To list zones in which it is available, run: + + gcloud compute machine-types list --filter="name=${var.machine_type}" + EOT + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml new file mode 100644 index 0000000000..95b6d1c730 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] +ghpc: + inject_module_id: name + has_to_be_used: true diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf new file mode 100644 index 0000000000..d3433bac43 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf @@ -0,0 +1,102 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "nodeset" { + description = "Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`." + value = local.nodeset + + precondition { + condition = !contains([ + "c3-:pd-standard", + "h3-:pd-standard", + "h3-:pd-ssd", + ], "${substr(var.machine_type, 0, 3)}:${var.disk_type}") + error_message = "A disk_type=${var.disk_type} cannot be used with machine_type=${var.machine_type}." + } + + precondition { + condition = var.reservation_name == "" || length(var.zones) == 0 + error_message = <<-EOD + If a reservation is specified, `var.zones` should be empty. + EOD + } + + precondition { + condition = var.placement_max_distance == null || var.enable_placement + error_message = "placement_max_distance requires enable_placement to be set to true." + } + + precondition { + condition = !(startswith(var.machine_type, "a3-") && var.placement_max_distance == 1) + error_message = "A3 machines do not support a placement_max_distance of 1." + } + + precondition { + condition = var.reservation_name == "" || !var.dws_flex.enabled + error_message = "Cannot use reservations with DWS Flex." + } + + precondition { + condition = !var.enable_placement || !var.dws_flex.enabled + error_message = "Cannot use DWS Flex with `enable_placement`." + } + + precondition { + condition = length(var.zones) == 0 || !var.dws_flex.enabled + error_message = <<-EOD + If a DWS Flex is enabled, `var.zones` should be empty. + EOD + } + + precondition { + condition = var.on_host_maintenance == "TERMINATE" || !var.dws_flex.enabled + error_message = "If DWS Flex is used, `on_host_maintenance` should be set to 'TERMINATE'" + } + + precondition { + condition = !var.enable_spot_vm || !var.dws_flex.enabled + error_message = "Cannot use both Flex-Start and Spot VMs for provisioning." + } + + precondition { + condition = var.reservation_name == "" || var.future_reservation == "" + error_message = "Cannot use reservations and future reservations in the same nodeset" + } + + precondition { + condition = !var.enable_placement || var.future_reservation == "" + error_message = "Cannot use `enable_placement` with future reservations." + } + + precondition { + condition = var.future_reservation == "" || length(var.zones) == 0 + error_message = <<-EOD + If a future reservation is specified, `var.zones` should be empty. + EOD + } + + precondition { + condition = var.future_reservation == "" || local.fr_zone == var.zone + error_message = <<-EOD + The zone of the deployment must match that of the future reservation + EOD + } + + precondition { + condition = var.node_count_dynamic_max > 0 || var.node_count_static > 0 + error_message = <<-EOD + This nodeset contains zero nodes, there should be at least one static or dynamic node + EOD + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf new file mode 100644 index 0000000000..0641cea7a8 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf @@ -0,0 +1,77 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # Currently supported images and projects + known_project_families = { + schedmd-slurm-public = [ + "slurm-gcp-6-9-debian-11", + "slurm-gcp-6-9-hpc-rocky-linux-8", + "slurm-gcp-6-9-ubuntu-2004-lts", + "slurm-gcp-6-9-ubuntu-2204-lts-arm64" + ] + } + + # This approach to "hacking" the project name allows a chain of Terraform + # calls to set the instance source_image (boot disk) with a "relative + # resource name" that passes muster with VPC Service Control rules + # + # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 + # https://cloud.google.com/apis/design/resource_names#relative_resource_name + source_image_project_normalized = (can(var.instance_image.family) ? + "projects/${data.google_compute_image.slurm.project}/global/images/family" : + "projects/${data.google_compute_image.slurm.project}/global/images" + ) + source_image_family = can(var.instance_image.family) ? data.google_compute_image.slurm.family : "" + source_image = can(var.instance_image.name) ? data.google_compute_image.slurm.name : "" +} + +data "google_compute_image" "slurm" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = var.instance_image.project + + lifecycle { + precondition { + condition = length(regexall("^projects/.+?/global/images/family$", var.instance_image.project)) == 0 + error_message = "The \"project\" field in var.instance_image no longer supports a long-form ending in \"family\". Specify only the project ID." + } + + postcondition { + condition = var.instance_image_custom || contains(keys(local.known_project_families), self.project) + error_message = <<-EOD + Images in project ${self.project} are not published by SchedMD. Images must be created by compatible releases of the Terraform and Packer modules following the guidance at https://goo.gle/hpc-slurm-images. Set var.instance_image_custom to true to silence this error and acknowledge that you are using a compatible image. + EOD + } + postcondition { + condition = !contains(keys(local.known_project_families), self.project) || try(contains(local.known_project_families[self.project], self.family), false) + error_message = <<-EOD + Image family ${self.family} published by SchedMD in project ${self.project} is not compatible with this release of the Terraform Slurm modules. Select from known compatible releases: + ${join("\n", [for p in try(local.known_project_families[self.project], []) : "\t\"${p}\""])} + EOD + } + postcondition { + condition = var.disk_size_gb >= self.disk_size_gb + error_message = "'disk_size_gb: ${var.disk_size_gb}' is smaller than the image size (${self.disk_size_gb}GB), please increase the blueprint disk size" + } + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf new file mode 100644 index 0000000000..643eca02c4 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf @@ -0,0 +1,635 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "name" { + description = <<-EOD + Name of the nodeset. Automatically populated by the module id if not set. + If setting manually, ensure a unique value across all nodesets. + EOD + type = string +} + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "node_conf" { + description = "Map of Slurm node line configuration." + type = map(any) + default = {} + validation { + condition = lookup(var.node_conf, "Sockets", null) == null + error_message = <<-EOD + `Sockets` field is in conflict with `SocketsPerBoard` which is automatically generated by SlurmGCP. + Instead, you can override the following fields: `Boards`, `SocketsPerBoard`, `CoresPerSocket`, and `ThreadsPerCore`. + See: https://slurm.schedmd.com/slurm.conf.html#OPT_Boards and https://slurm.schedmd.com/slurm.conf.html#OPT_Sockets_1 + EOD + } +} + +variable "node_count_static" { + description = "Number of nodes to be statically created." + type = number + default = 0 +} + +variable "node_count_dynamic_max" { + description = "Maximum number of auto-scaling nodes allowed in this partition." + type = number + default = 10 +} + +## VM Definition +variable "instance_template" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: Instance template can not be specified for compute nodes." + type = string + default = null + validation { + condition = var.instance_template == null + error_message = "DEPRECATED: Instance template can not be specified for compute nodes." + } +} + +variable "machine_type" { + description = "Compute Platform machine type to use for this partition compute nodes." + type = string + default = "c2-standard-60" +} + +variable "metadata" { + type = map(string) + description = "Metadata, provided as a map." + default = {} +} + +variable "instance_image" { + description = <<-EOD + Defines the image that will be used in the Slurm node group VM instances. + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + + For more information on creating custom images that comply with Slurm on GCP + see the "Slurm on GCP Custom Images" section in docs/vm-images.md. + EOD + type = map(string) + default = { + family = "slurm-gcp-6-9-hpc-rocky-linux-8" + project = "schedmd-slurm-public" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "instance_image_custom" { + description = <<-EOD + A flag that designates that the user is aware that they are requesting + to use a custom and potentially incompatible image for this Slurm on + GCP module. + + If the field is set to false, only the compatible families and project + names will be accepted. The deployment will fail with any other image + family or name. If set to true, no checks will be done. + + See: https://goo.gle/hpc-slurm-images + EOD + type = bool + default = false +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "tags" { + type = list(string) + description = "Network tag list." + default = [] +} + +variable "disk_type" { + description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." + type = string + default = "pd-standard" +} + +variable "disk_size_gb" { + description = "Size of boot disk to create for the partition compute nodes." + type = number + default = 50 +} + +variable "disk_auto_delete" { + type = bool + description = "Whether or not the boot disk should be auto-deleted." + default = true +} + +variable "disk_labels" { + description = "Labels specific to the boot disk. These will be merged with var.labels." + type = map(string) + default = {} +} + +variable "disk_resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} + validation { + condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) + error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" + } + validation { + condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) + error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" + } +} + +variable "additional_disks" { + description = "Configurations of additional disks to be included on the partition nodes." + type = list(object({ + disk_name = optional(string) + device_name = optional(string) + disk_size_gb = optional(number) + disk_type = optional(string) + disk_labels = optional(map(string)) + auto_delete = optional(bool) + boot = optional(bool) + disk_resource_manager_tags = optional(map(string)) + })) + default = [] +} + +variable "enable_confidential_vm" { + type = bool + description = "Enable the Confidential VM configuration. Note: the instance image must support option." + default = false +} + +variable "enable_shielded_vm" { + type = bool + description = "Enable the Shielded VM configuration. Note: the instance image must support option." + default = false +} + +variable "shielded_instance_config" { + type = object({ + enable_integrity_monitoring = bool + enable_secure_boot = bool + enable_vtpm = bool + }) + description = <<-EOD + Shielded VM configuration for the instance. Note: not used unless + enable_shielded_vm is 'true'. + - enable_integrity_monitoring : Compare the most recent boot measurements to the + integrity policy baseline and return a pair of pass/fail results depending on + whether they match or not. + - enable_secure_boot : Verify the digital signature of all boot components, and + halt the boot process if signature verification fails. + - enable_vtpm : Use a virtualized trusted platform module, which is a + specialized computer chip you can use to encrypt objects like keys and + certificates. + EOD + default = { + enable_integrity_monitoring = true + enable_secure_boot = true + enable_vtpm = true + } +} + + +variable "enable_oslogin" { + type = bool + description = <<-EOD + Enables Google Cloud os-login for user login and authentication for VMs. + See https://cloud.google.com/compute/docs/oslogin + EOD + default = true +} + +variable "can_ip_forward" { + description = "Enable IP forwarding, for NAT instances for example." + type = bool + default = false +} + +variable "advanced_machine_features" { + description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" + type = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) + default = { + threads_per_core = 1 # disable SMT by default + } +} + +variable "resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} + validation { + condition = alltrue([for value in var.resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) + error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" + } + validation { + condition = alltrue([for value in keys(var.resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) + error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" + } +} + +variable "enable_smt" { # tflint-ignore: terraform_unused_declarations + type = bool + description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + default = null + validation { + condition = var.enable_smt == null + error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + } +} + +variable "labels" { + description = "Labels to add to partition compute instances. Key-value pairs." + type = map(string) + default = {} +} + +variable "min_cpu_platform" { + description = "The name of the minimum CPU platform that you want the instance to use." + type = string + default = null +} + +variable "on_host_maintenance" { + type = string + description = <<-EOD + Instance availability Policy. + + Note: Placement groups are not supported when on_host_maintenance is set to + "MIGRATE" and will be deactivated regardless of the value of + enable_placement. To support enable_placement, ensure on_host_maintenance is + set to "TERMINATE". + EOD + default = "TERMINATE" +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = string, + count = number + })) + default = [] + nullable = false + + validation { + condition = length(var.guest_accelerator) <= 1 + error_message = "The Slurm modules supports 0 or 1 models of accelerator card on each node." + } +} + +variable "preemptible" { + description = "Should use preemptibles to burst." + type = bool + default = false +} + + +variable "service_account_email" { + description = "Service account e-mail address to attach to the compute instances." + type = string + default = null +} + +variable "service_account_scopes" { + description = "Scopes to attach to the compute instances." + type = set(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +variable "service_account" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." + type = object({ + email = string + scopes = set(string) + }) + default = null + validation { + condition = var.service_account == null + error_message = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." + } +} + +variable "enable_spot_vm" { + description = "Enable the partition to use spot VMs (https://cloud.google.com/spot-vms)." + type = bool + default = false +} + +variable "spot_instance_config" { + description = "Configuration for spot VMs." + type = object({ + termination_action = string + }) + default = null +} + +variable "bandwidth_tier" { + description = < 0 + error_message = "Reservation name must be either empty or in the format '[projects/PROJECT_ID/reservations/]RESERVATION_NAME[/reservationBlocks/BLOCK_ID]', [...] are optional parts." + } +} + +variable "future_reservation" { + description = <<-EOD + If set, will make use of the future reservation for the nodeset. Input can be either the future reservation name or its selfLink in the format 'projects/PROJECT_ID/zones/ZONE/futureReservations/FUTURE_RESERVATION_NAME'. + See https://cloud.google.com/compute/docs/instances/future-reservations-overview + EOD + type = string + default = "" + nullable = false + + validation { + condition = length(regexall("^(projects/([a-z0-9-]+)/zones/([a-z0-9-]+)/futureReservations/([a-z0-9-]+))?$", var.future_reservation)) > 0 || length(regexall("^([a-z0-9-]+)$", var.future_reservation)) > 0 + error_message = "Future reservation must be either the future reservation name or its selfLink in the format 'projects/PROJECT_ID/zone/ZONE/futureReservations/FUTURE_RESERVATION_NAME'." + } +} + +variable "maintenance_interval" { + description = <<-EOD + Sets the maintenance interval for instances in this nodeset. + See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#maintenance_interval. + EOD + type = string + default = null +} + +variable "startup_script" { + description = "Startup script used by VMs in this nodeset" + type = string + default = "# no-op" +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured on nodes." + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + })) + default = [] +} + + +variable "instance_properties" { + description = <<-EOD + Override the instance properties. Used to test features not supported by Slurm GCP, + recommended for advanced usage only. + See https://cloud.google.com/compute/docs/reference/rest/v1/regionInstances/bulkInsert + If any sub-field (e.g. scheduling) is set, it will override the values computed by + SlurmGCP and ignoring values of provided vars. + EOD + type = any + default = null +} + + +variable "enable_maintenance_reservation" { + type = bool + description = "Enables slurm reservation for scheduled maintenance." + default = false +} + + +variable "enable_opportunistic_maintenance" { + type = bool + description = "On receiving maintenance notification, maintenance will be performed as soon as nodes becomes idle." + default = false +} + + +variable "dws_flex" { + description = <<-EOD + If set and `enabled = true`, will utilize the DWS Flex Start to provision nodes. + See: https://cloud.google.com/blog/products/compute/introducing-dynamic-workload-scheduler + Options: + - enable: Enable DWS Flex Start + - max_run_duration: Maximum duration in seconds for the job to run, should not exceed 604,800 (one week). + - use_job_duration: Use the job duration to determine the max_run_duration, if job duration is not set, max_run_duration will be used. + - use_bulk_insert: Uses the legacy implementation of DWS Flex Start with Bulk Insert for non-accelerator instances + + Limitations: + - CAN NOT be used with reservations; + - CAN NOT be used with placement groups; + - If `use_job_duration` is enabled nodeset can be used in "exclusive" partitions only + + EOD + + type = object({ + enabled = optional(bool, true) + max_run_duration = optional(number, 604800) # one week + use_job_duration = optional(bool, false) + use_bulk_insert = optional(bool, false) + }) + default = { + enabled = false + } + validation { + condition = var.dws_flex.max_run_duration >= 600 && var.dws_flex.max_run_duration <= 604800 + error_message = "Max duration must be at least than 10 minutes, and cannot be more than one week." + } +} + +variable "placement_max_distance" { + type = number + description = "Maximum distance between nodes in the placement group. Requires enable_placement to be true. Values must be supported by the chosen machine type." + nullable = true + default = null + + validation { + condition = coalesce(var.placement_max_distance, 1) >= 1 && coalesce(var.placement_max_distance, 3) <= 3 + error_message = "Invalid value for placement_max_distance. Valid values are null, 1, 2, or 3." + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf new file mode 100644 index 0000000000..b879e1bd17 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.4" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 5.11" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset/v1.57.0" + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md new file mode 100644 index 0000000000..61167a9f1e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md @@ -0,0 +1,105 @@ +## Description + +This module creates a compute partition that can be used as input to the +[schedmd-slurm-gcp-v6-controller](../../scheduler/schedmd-slurm-gcp-v6-controller/README.md). + +The partition module is designed to work alongside the +[schedmd-slurm-gcp-v6-nodeset](../schedmd-slurm-gcp-v6-nodeset/README.md) +module. A partition can be made up of one or +more nodesets, provided either through `use` (preferred) or defined manually +in the `nodeset` variable. + +### Example + +The following code snippet creates a partition module with: + +* 2 nodesets added via `use`. + * The first nodeset is made up of machines of type `c2-standard-30`. + * The second nodeset is made up of machines of type `c2-standard-60`. + * Both nodesets have a maximum count of 200 dynamically created nodes. +* partition name of "compute". +* connected to the `network` module via `use`. +* nodes mounted to homefs via `use`. + +```yaml +- id: nodeset_1 + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: + - network + settings: + name: c30 + node_count_dynamic_max: 200 + machine_type: c2-standard-30 + +- id: nodeset_2 + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: + - network + settings: + name: c60 + node_count_dynamic_max: 200 + machine_type: c2-standard-60 + +- id: compute_partition + source: community/modules/compute/schedmd-slurm-gcp-v6-partition + use: + - homefs + - nodeset_1 + - nodeset_2 + settings: + partition_name: compute +``` + +## Support + +The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform +modules. For support with the underlying modules, see the instructions in the +[slurm-gcp README][slurm-gcp-readme]. + +[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp +[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [exclusive](#input\_exclusive) | Exclusive job access to nodes. When set to true nodes execute single job and are deleted
after job exits. If set to false, multiple jobs can be scheduled on one node. | `bool` | `true` | no | +| [is\_default](#input\_is\_default) | Sets this partition as the default partition by updating the partition\_conf.
If "Default" is already set in partition\_conf, this variable will have no effect. | `bool` | `false` | no | +| [network\_storage](#input\_network\_storage) | DEPRECATED | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [nodeset](#input\_nodeset) | A list of nodesets.
For type definition see community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf::nodeset | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 1)
    node_conf              = optional(map(string), {})
    nodeset_name           = string
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    bandwidth_tier                   = optional(string, "platform_default")
    can_ip_forward                   = optional(bool, false)
    disk_auto_delete                 = optional(bool, true)
    disk_labels                      = optional(map(string), {})
    disk_resource_manager_tags       = optional(map(string), {})
    disk_size_gb                     = optional(number)
    disk_type                        = optional(string)
    enable_confidential_vm           = optional(bool, false)
    enable_placement                 = optional(bool, false)
    placement_max_distance           = optional(number, null)
    enable_oslogin                   = optional(bool, true)
    enable_shielded_vm               = optional(bool, false)
    enable_maintenance_reservation   = optional(bool, false)
    enable_opportunistic_maintenance = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    dws_flex = object({
      enabled          = bool
      max_run_duration = number
      use_job_duration = bool
      use_bulk_insert  = bool
    })
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    maintenance_interval     = optional(string)
    instance_properties_json = string
    metadata                 = optional(map(string), {})
    min_cpu_platform         = optional(string)
    network_tier             = optional(string, "STANDARD")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    subnetwork_self_link = string
    additional_networks = optional(list(object({
      network            = string
      subnetwork         = string
      subnetwork_project = string
      network_ip         = string
      nic_type           = string
      stack_type         = string
      queue_count        = number
      access_config = list(object({
        nat_ip       = string
        network_tier = string
      }))
      ipv6_access_config = list(object({
        network_tier = string
      }))
      alias_ip_range = list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      }))
    })))
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    spot               = optional(bool, false)
    tags               = optional(list(string), [])
    termination_action = optional(string)
    reservation_name   = optional(string)
    future_reservation = string
    startup_script = optional(list(object({
      filename = string
    content = string })), [])

    zone_target_shape = string
    zone_policy_allow = set(string)
    zone_policy_deny  = set(string)
  }))
| `[]` | no | +| [nodeset\_dyn](#input\_nodeset\_dyn) | Defines dynamic nodesets, as a list. | 
list(object({
    nodeset_name    = string
    nodeset_feature = string
  }))
| `[]` | no | +| [nodeset\_tpu](#input\_nodeset\_tpu) | Define TPU nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 5)
    nodeset_name           = string
    enable_public_ip       = optional(bool, false)
    node_type              = string
    accelerator_config = optional(object({
      topology = string
      version  = string
      }), {
      topology = ""
      version  = ""
    })
    tf_version   = string
    preemptible  = optional(bool, false)
    preserve_tpu = optional(bool, false)
    zone         = string
    data_disks   = optional(list(string), [])
    docker_image = optional(string, "")
    network_storage = optional(list(object({
      server_ip     = string
      remote_mount  = string
      local_mount   = string
      fs_type       = string
      mount_options = string
    })), [])
    subnetwork = string
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    project_id = string
    reserved   = optional(string, false)
  }))
| `[]` | no | +| [partition\_conf](#input\_partition\_conf) | Slurm partition configuration as a map.
See https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION | `map(string)` | `{}` | no | +| [partition\_name](#input\_partition\_name) | The name of the slurm partition. | `string` | n/a | yes | +| [resume\_timeout](#input\_resume\_timeout) | Maximum time permitted (in seconds) between when a node resume request is issued and when the node is actually available for use.
If null is given, then a smart default will be chosen depending on nodesets in partition.
This sets 'ResumeTimeout' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_ResumeTimeout_1 for details. | `number` | `null` | no | +| [suspend\_time](#input\_suspend\_time) | Nodes which remain idle or down for this number of seconds will be placed into power save mode by SuspendProgram.
This sets 'SuspendTime' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTime_1 for details.
NOTE: use value -1 to exclude partition from suspend.
NOTE 2: if `var.exclusive` is set to true (default), nodes are deleted immediately after job finishes. | `number` | `300` | no | +| [suspend\_timeout](#input\_suspend\_timeout) | Maximum time permitted (in seconds) between when a node suspend request is issued and when the node is shutdown.
If null is given, then a smart default will be chosen depending on nodesets in partition.
This sets 'SuspendTimeout' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTimeout_1 for details. | `number` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [nodeset](#output\_nodeset) | Details of a nodesets in this partition | +| [nodeset\_dyn](#output\_nodeset\_dyn) | Details of a dynamic nodesets in this partition | +| [nodeset\_tpu](#output\_nodeset\_tpu) | Details of a TPU nodesets in this partition | +| [partitions](#output\_partitions) | Details of a slurm partition | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf new file mode 100644 index 0000000000..35b4ca3c19 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf @@ -0,0 +1,42 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + use_static = [for ns in concat(var.nodeset, var.nodeset_tpu) : ns.nodeset_name if ns.node_count_static > 0] + uses_job_duration = length([for ns in var.nodeset : ns.dws_flex.use_job_duration if ns.dws_flex.use_job_duration]) > 0 + + has_node = length(var.nodeset) > 0 + has_dyn = length(var.nodeset_dyn) > 0 + has_tpu = length(var.nodeset_tpu) > 0 + has_flex = length([for ns in var.nodeset : ns.dws_flex.enabled if ns.dws_flex.enabled]) > 0 +} + +locals { + partition_conf = merge({ + "Default" = var.is_default ? "YES" : null + "SuspendTime" = var.suspend_time < 0 ? "INFINITE" : var.suspend_time + "SuspendTimeout" = var.suspend_timeout != null ? var.suspend_timeout : (local.has_tpu ? 240 : 120) + }, var.partition_conf, { "ResumeTimeout" = local.has_flex ? 65535 : try(var.partition_conf["ResumeTimeout"], coalesce(var.resume_timeout, (local.has_tpu ? 600 : 300))) }) + + partition = { + partition_name = var.partition_name + partition_conf = local.partition_conf + + partition_nodeset = [for ns in var.nodeset : ns.nodeset_name] + partition_nodeset_tpu = [for ns in var.nodeset_tpu : ns.nodeset_name] + partition_nodeset_dyn = [for ns in var.nodeset_dyn : ns.nodeset_name] + # Options + enable_job_exclusive = var.exclusive + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml new file mode 100644 index 0000000000..13ea127b3c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] +ghpc: + has_to_be_used: true diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf new file mode 100644 index 0000000000..ff796ad0f6 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf @@ -0,0 +1,59 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "partitions" { + description = "Details of a slurm partition" + + value = [local.partition] + + precondition { + condition = (length(local.use_static) == 0) || !var.exclusive + error_message = <<-EOD + Can't use static nodes within partition with `var.exclusive` set to `true`. + NOTE: Partition's `var.exclusive` is set to `true` by default. Set it to `false` explicitly to use static nodes. + EOD + } + + precondition { + # Can not mix TPU with other non-TPU nodesets due to SlurmGCP specific limitations; + # Can not mix dynamic with non-dynamic nodesets due to Slurms inability to + # turn off "power management" at nodeset level (can only do it at partition or node level). + condition = sum([for b in [local.has_node, local.has_dyn, local.has_tpu] : b ? 1 : 0]) == 1 + error_message = "Partition must contain exactly one type of nodeset." + } + + precondition { + condition = !local.uses_job_duration || var.exclusive + error_message = "`use_job_duration` can only be used in exclusive partitions" + } +} + +output "nodeset" { + description = "Details of a nodesets in this partition" + + value = var.nodeset +} + +output "nodeset_tpu" { + description = "Details of a TPU nodesets in this partition" + + value = var.nodeset_tpu +} + + +output "nodeset_dyn" { + description = "Details of a dynamic nodesets in this partition" + + value = var.nodeset_dyn +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf new file mode 100644 index 0000000000..338d90954f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf @@ -0,0 +1,310 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "partition_name" { + description = "The name of the slurm partition." + type = string + + validation { + condition = can(regex("^[a-z](?:[a-z0-9]*)$", var.partition_name)) + error_message = "Variable 'partition_name' must be a match of regex '^[a-z](?:[a-z0-9]*)$'." + } +} + +variable "partition_conf" { + description = <<-EOD + Slurm partition configuration as a map. + See https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION + EOD + type = map(string) + default = {} +} + +variable "is_default" { + description = <<-EOD + Sets this partition as the default partition by updating the partition_conf. + If "Default" is already set in partition_conf, this variable will have no effect. + EOD + type = bool + default = false +} + +variable "exclusive" { + description = <<-EOD + Exclusive job access to nodes. When set to true nodes execute single job and are deleted + after job exits. If set to false, multiple jobs can be scheduled on one node. + EOD + type = bool + default = true +} + +variable "nodeset" { + description = <<-EOD + A list of nodesets. + For type definition see community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf::nodeset + EOD + type = list(object({ + node_count_static = optional(number, 0) + node_count_dynamic_max = optional(number, 1) + node_conf = optional(map(string), {}) + nodeset_name = string + additional_disks = optional(list(object({ + disk_name = optional(string) + device_name = optional(string) + disk_size_gb = optional(number) + disk_type = optional(string) + disk_labels = optional(map(string), {}) + auto_delete = optional(bool, true) + boot = optional(bool, false) + disk_resource_manager_tags = optional(map(string), {}) + })), []) + bandwidth_tier = optional(string, "platform_default") + can_ip_forward = optional(bool, false) + disk_auto_delete = optional(bool, true) + disk_labels = optional(map(string), {}) + disk_resource_manager_tags = optional(map(string), {}) + disk_size_gb = optional(number) + disk_type = optional(string) + enable_confidential_vm = optional(bool, false) + enable_placement = optional(bool, false) + placement_max_distance = optional(number, null) + enable_oslogin = optional(bool, true) + enable_shielded_vm = optional(bool, false) + enable_maintenance_reservation = optional(bool, false) + enable_opportunistic_maintenance = optional(bool, false) + gpu = optional(object({ + count = number + type = string + })) + dws_flex = object({ + enabled = bool + max_run_duration = number + use_job_duration = bool + use_bulk_insert = bool + }) + labels = optional(map(string), {}) + machine_type = optional(string) + advanced_machine_features = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) + maintenance_interval = optional(string) + instance_properties_json = string + metadata = optional(map(string), {}) + min_cpu_platform = optional(string) + network_tier = optional(string, "STANDARD") + network_storage = optional(list(object({ + server_ip = string + remote_mount = string + local_mount = string + fs_type = string + mount_options = string + client_install_runner = optional(map(string)) + mount_runner = optional(map(string)) + })), []) + on_host_maintenance = optional(string) + preemptible = optional(bool, false) + region = optional(string) + resource_manager_tags = optional(map(string), {}) + service_account = optional(object({ + email = optional(string) + scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) + })) + shielded_instance_config = optional(object({ + enable_integrity_monitoring = optional(bool, true) + enable_secure_boot = optional(bool, true) + enable_vtpm = optional(bool, true) + })) + source_image_family = optional(string) + source_image_project = optional(string) + source_image = optional(string) + subnetwork_self_link = string + additional_networks = optional(list(object({ + network = string + subnetwork = string + subnetwork_project = string + network_ip = string + nic_type = string + stack_type = string + queue_count = number + access_config = list(object({ + nat_ip = string + network_tier = string + })) + ipv6_access_config = list(object({ + network_tier = string + })) + alias_ip_range = list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })) + }))) + access_config = optional(list(object({ + nat_ip = string + network_tier = string + }))) + spot = optional(bool, false) + tags = optional(list(string), []) + termination_action = optional(string) + reservation_name = optional(string) + future_reservation = string + startup_script = optional(list(object({ + filename = string + content = string })), []) + + zone_target_shape = string + zone_policy_allow = set(string) + zone_policy_deny = set(string) + })) + default = [] + + validation { + condition = length(distinct(var.nodeset[*].nodeset_name)) == length(var.nodeset) + error_message = "All nodesets must have a unique name." + } +} + +variable "nodeset_tpu" { + description = "Define TPU nodesets, as a list." + type = list(object({ + node_count_static = optional(number, 0) + node_count_dynamic_max = optional(number, 5) + nodeset_name = string + enable_public_ip = optional(bool, false) + node_type = string + accelerator_config = optional(object({ + topology = string + version = string + }), { + topology = "" + version = "" + }) + tf_version = string + preemptible = optional(bool, false) + preserve_tpu = optional(bool, false) + zone = string + data_disks = optional(list(string), []) + docker_image = optional(string, "") + network_storage = optional(list(object({ + server_ip = string + remote_mount = string + local_mount = string + fs_type = string + mount_options = string + })), []) + subnetwork = string + service_account = optional(object({ + email = optional(string) + scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) + })) + project_id = string + reserved = optional(string, false) + })) + default = [] + + validation { + condition = length(distinct([for x in var.nodeset_tpu : x.nodeset_name])) == length(var.nodeset_tpu) + error_message = "All TPU nodesets must have a unique name." + } +} + +variable "nodeset_dyn" { + description = "Defines dynamic nodesets, as a list." + type = list(object({ + nodeset_name = string + nodeset_feature = string + })) + default = [] + + validation { + condition = length(distinct([for x in var.nodeset_dyn : x.nodeset_name])) == length(var.nodeset_dyn) + error_message = "All dynamic nodesets must have a unique name." + } +} + +variable "resume_timeout" { + description = <<-EOD + Maximum time permitted (in seconds) between when a node resume request is issued and when the node is actually available for use. + If null is given, then a smart default will be chosen depending on nodesets in partition. + This sets 'ResumeTimeout' in partition_conf. + See https://slurm.schedmd.com/slurm.conf.html#OPT_ResumeTimeout_1 for details. + EOD + type = number + default = null + + validation { + condition = var.resume_timeout == null ? true : var.resume_timeout > 0 && var.resume_timeout < 65536 + error_message = "Value must be > 0 and < 65536" + } +} + +variable "suspend_time" { + description = <<-EOD + Nodes which remain idle or down for this number of seconds will be placed into power save mode by SuspendProgram. + This sets 'SuspendTime' in partition_conf. + See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTime_1 for details. + NOTE: use value -1 to exclude partition from suspend. + NOTE 2: if `var.exclusive` is set to true (default), nodes are deleted immediately after job finishes. + EOD + type = number + default = 300 + + validation { + condition = var.suspend_time >= -1 + error_message = "Value must be >= -1." + } +} + +variable "suspend_timeout" { + description = <<-EOD + Maximum time permitted (in seconds) between when a node suspend request is issued and when the node is shutdown. + If null is given, then a smart default will be chosen depending on nodesets in partition. + This sets 'SuspendTimeout' in partition_conf. + See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTimeout_1 for details. + EOD + type = number + default = null + + validation { + condition = var.suspend_timeout == null ? true : var.suspend_timeout > 0 + error_message = "Value must be > 0." + } +} + + +# tflint-ignore: terraform_unused_declarations +variable "network_storage" { + description = "DEPRECATED" + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] + validation { + condition = length(var.network_storage) == 0 + error_message = <<-EOD + network_storage in partition module is deprecated and should not be set. + To add network storage to compute nodes, use network_storage of nodeset module instead. + EOD + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf new file mode 100644 index 0000000000..5298c89dbe --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.3" + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-partition/v1.57.0" + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/README.md new file mode 100644 index 0000000000..994f1500ba --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/README.md @@ -0,0 +1,157 @@ +## Description + +This module provides ways to create and manage Google Cloud Artifact Registry repositories. + +Currently this module is built to support repositories in Docker format although there are placeholder variables for other types which may work too. Remote repositories with pull-through cache functionality integrated with Google Secret Manager is currently supported. The aim of this module is to eventually offer feature parity with this [Terraform module](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/artifact_registry_repository#nested_remote_repository_config), allowing creation of repositories in various formats, including Docker, Maven, NPM, Python, APT, YUM, and COMMON. + +This module is best suited for managing artifact repositories in HPC/AI containerized environments where artifacts need to be shared across distributed systems. It includes IAM role configurations and secret access handling for seamless integration with CI/CD pipelines and other services too. + +It is designed to help facilitate containerized workloads running in the Cluster Toolkit with SLURM leveraging [Enroot](https://github.com/NVIDIA/enroot) and [Pyxis](https://github.com/NVIDIA/pyxis). Docker repositories can store container images that are used in job submissions, enabling efficient and scalable execution of containerized HPC or AI based workloads. + +## Usage + +### Service Account / APIs + +You will need to enable the relevant APIs and create a Service Account for your cluster with the following Artifact Registry permissions. + +```yaml + - id: services-api + source: community/modules/project/service-enablement + settings: + gcp_service_list: + - secretmanager.googleapis.com + - cloudbuild.googleapis.com + - artifactregistry.googleapis.com + + - source: community/modules/project/service-account + kind: terraform + id: hpc_service_account + settings: + project_id: project_name + name: service_account_name + project_roles: + - artifactregistry.reader + - artifactregistry.writer + - secretmanager.secretAccessor +``` + +### Deployment + +Create a standard Docker repository. + +```yaml +- id: registry + source: community/modules/container/artifact-registry + settings: + repo_mode: STANDARD_REPOSITORY + format: DOCKER +``` + +Mirror of public Docker Hub repository. + +```yaml +- id: dockerhub_registry + source: community/modules/container/artifact-registry + settings: + repo_mode: REMOTE_REPOSITORY + format: DOCKER + repo_public_repository: DOCKER_HUB +``` + +Mirror of NVIDIA's [NGC Catalog](https://catalog.ngc.nvidia.com/containers). [API key](https://org.ngc.nvidia.com/setup/api-key) used in blueprint is stored in Secret Manager. + +```yaml +- id: ngc_registry + source: community/modules/container/artifact-registry + settings: + repo_mode: REMOTE_REPOSITORY + format: DOCKER + repo_mirror_url: "https://nvcr.io" + repo_username: $oauthtoken + repo_password: api_key_here + use_upstream_credentials: True +``` + +### Container Operations + +Retrieve `$REPOSITORY_NAME` from [Artifact Registry](https://console.cloud.google.com/artifacts) or by using `gcloud`. + +```yaml +gcloud artifacts repositories list --project="${PROJECT_ID}" +``` + +Pulling containers from your mirrored internal Artifact Repositories. + +Pull [Ubuntu](https://hub.docker.com/_/ubuntu) from Docker Hub mirror. + +```yaml +docker pull ${REGION}-docker.pkg.dev/${PROJECT_NAME}/${REPOSITORY_NAME}/library/ubuntu:latest +``` + +Pull [Pytorch](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch) from NGC Catalog mirror. + +```yaml +docker pull ${REGION}-docker.pkg.dev/${PROJECT_NAME}/${REPOSITORY_NAME}/nvidia/pytorch:24.11-py3 +``` + +Alternatively, proceed with running SLURM's [NVIDIA/pyxis](https://github.com/NVIDIA/pyxis) plugin, which will now be able to pull and use these containers directly from the mirrored repositories. + +Note: only Docker registries have been tested so far. Placeholders do exist for other registry types which may or may not work. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [random](#provider\_random) | ~> 3.0 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_artifact_registry_repository.artifact_registry](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/artifact_registry_repository) | resource | +| [google_secret_manager_secret.repo_password_secret](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | +| [google_secret_manager_secret_version.repo_password_secret_version](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [random_password.repo_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | +| [terraform_data.input_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment. | `string` | n/a | yes | +| [format](#input\_format) | Artifact Registry format (e.g., DOCKER). | `string` | `"DOCKER"` | no | +| [labels](#input\_labels) | Labels to add to the artifact registry. Key-value pairs. | `map(string)` | `{}` | no | +| [project\_id](#input\_project\_id) | Project ID where the artifact registry and secret are created. | `string` | n/a | yes | +| [region](#input\_region) | Region for the artifact registry. | `string` | n/a | yes | +| [repo\_mirror\_url](#input\_repo\_mirror\_url) | For REMOTE\_REPOSITORY, URL for a custom or common mirror. | `string` | `null` | no | +| [repo\_mode](#input\_repo\_mode) | Artifact Registry mode (STANDARD\_REPOSITORY, REMOTE\_REPOSITORY, etc.). | `string` | `"STANDARD_REPOSITORY"` | no | +| [repo\_password](#input\_repo\_password) | Optional password/API key. If null, one will be randomly generated. | `string` | `null` | no | +| [repo\_public\_repository](#input\_repo\_public\_repository) | For REMOTE\_REPOSITORY, name of a known public repo as per the Terraform module
(e.g., DOCKER\_HUB) or null for custom repo. | `string` | `null` | no | +| [repo\_username](#input\_repo\_username) | Username for external repository. | `string` | `null` | no | +| [repository\_base](#input\_repository\_base) | For APT/YUM public repos, repository\_base (e.g., 'DEBIAN', 'UBUNTU'). | `string` | `null` | no | +| [repository\_path](#input\_repository\_path) | For APT/YUM public repos, repository\_path (e.g., 'debian/dists/buster'). | `string` | `null` | no | +| [use\_upstream\_credentials](#input\_use\_upstream\_credentials) | Configure Service Account to use upstream credentials for REMOTE\_REPOSITORY:
If true, a username/password is used for the REMOTE\_REPOSITORY mirror.
If false (or if repo\_password == null), no password is created at all.
Note: Blueprint credentials will be stored in Secrets Manager. | `bool` | `false` | no | +| [user\_managed\_replication](#input\_user\_managed\_replication) | (Optional) A list of objects to enable user-managed replication.
Each object can have:
location = string
kms\_key\_name = optional(string)
If empty, auto replication is used. | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [registry\_url](#output\_registry\_url) | The URL of the created artifact registry. | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/main.tf new file mode 100644 index 0000000000..c3406af607 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/main.tf @@ -0,0 +1,268 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "artifact-registry", ghpc_role = "container" }) +} + +locals { + # Auto (i.e., empty) vs user-managed replication + auto = length(var.user_managed_replication) == 0 ? true : false + + # For remote custom repositories, parse out host to create a base_component name + mirror_url_no_proto = var.repo_mirror_url != null ? replace(replace(var.repo_mirror_url, "https://", ""), "http://", "") : "" + mirror_host = local.mirror_url_no_proto != "" ? split("/", local.mirror_url_no_proto)[0] : "" + + base_component = replace( + replace( + replace( + lower( + local.mirror_host != "" + ? "${var.format}-${var.repo_mode}-${local.mirror_host}" + : "${var.format}-${var.repo_mode}-nohost" + ), + "\\.", "-" + ), + "/", "-" + ), + "_", "-" + ) + + repository_suffix = random_id.resource_name_suffix.hex + + # The final name for the artifact registry repository + repository_name = replace( + replace( + lower( + format("%s-%s", local.base_component, local.repository_suffix) + ), + ".", "-" + ), + "/", "-" + ) + + # The secret name is derived from the repository name + # with a suffix like "-secret". + derived_secret_name = format("%s-secret", local.repository_name) +} + +############################## +# PASSWORD / SECRET +############################## + +# Only create a random password if user didn't supply one +resource "random_password" "repo_password" { + count = var.use_upstream_credentials && var.repo_password == null ? 1 : 0 + length = 24 + special = true + override_special = "_-#=." +} + +resource "google_secret_manager_secret" "repo_password_secret" { + count = var.use_upstream_credentials ? 1 : 0 + project = var.project_id + + # Derive the secret ID from the repository name + secret_id = local.derived_secret_name + + labels = local.labels + + replication { + dynamic "auto" { + for_each = local.auto ? [1] : [] + content {} + } + dynamic "user_managed" { + for_each = local.auto ? [] : [1] + content { + dynamic "replicas" { + for_each = var.user_managed_replication + content { + location = replicas.value.location + dynamic "customer_managed_encryption" { + for_each = replicas.value.kms_key_name != null ? [1] : [] + content { + kms_key_name = customer_managed_encryption.value + } + } + } + } + } + } + } +} + +resource "google_secret_manager_secret_version" "repo_password_secret_version" { + count = var.use_upstream_credentials ? 1 : 0 + secret = google_secret_manager_secret.repo_password_secret[0].id + + # If user provided a password, use it. Otherwise use the random password. + secret_data = var.repo_password != null ? var.repo_password : random_password.repo_password[0].result +} + +############################## +# IAM BINDINGS +############################## + +############################## +# ARTIFACT REGISTRY +############################## + +resource "random_id" "resource_name_suffix" { + byte_length = 2 +} + +resource "google_artifact_registry_repository" "artifact_registry" { + project = var.project_id + location = var.region + format = var.format + mode = var.repo_mode + description = var.deployment_name + labels = local.labels + repository_id = local.repository_name + + # Only create remote_repository_config if REMOTE_REPOSITORY + dynamic "remote_repository_config" { + for_each = var.repo_mode == "REMOTE_REPOSITORY" ? [1] : [] + content { + description = "Pull-through cache" + + dynamic "docker_repository" { + for_each = var.format == "DOCKER" && var.repo_public_repository != null ? [1] : [] + content { + public_repository = var.repo_public_repository + } + } + + dynamic "docker_repository" { + for_each = var.format == "DOCKER" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "maven_repository" { + for_each = var.format == "MAVEN" && var.repo_public_repository != null ? [1] : [] + content { + public_repository = var.repo_public_repository + } + } + + dynamic "maven_repository" { + for_each = var.format == "MAVEN" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "npm_repository" { + for_each = var.format == "NPM" && var.repo_public_repository != null ? [1] : [] + content { + public_repository = var.repo_public_repository + } + } + + dynamic "npm_repository" { + for_each = var.format == "NPM" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "python_repository" { + for_each = var.format == "PYTHON" && var.repo_public_repository != null ? [1] : [] + content { + public_repository = var.repo_public_repository + } + } + + dynamic "python_repository" { + for_each = var.format == "PYTHON" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "apt_repository" { + for_each = var.format == "APT" && var.repo_public_repository != null ? [1] : [] + content { + public_repository { + repository_base = var.repository_base + repository_path = var.repository_path + } + } + } + + dynamic "apt_repository" { + for_each = var.format == "APT" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "yum_repository" { + for_each = var.format == "YUM" && var.repo_public_repository != null ? [1] : [] + content { + public_repository { + repository_base = var.repository_base + repository_path = var.repository_path + } + } + } + + dynamic "yum_repository" { + for_each = var.format == "YUM" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] + content { + custom_repository { + uri = var.repo_mirror_url + } + } + } + + dynamic "common_repository" { + for_each = var.format == "COMMON" ? [1] : [] + content { + uri = var.repo_mirror_url + } + } + + # Only enable upstream credentials if user wants it + dynamic "upstream_credentials" { + for_each = var.use_upstream_credentials ? [1] : [] + content { + username_password_credentials { + username = var.repo_username + password_secret_version = google_secret_manager_secret_version.repo_password_secret_version[0].name + } + } + } + } + } + + depends_on = [ + google_secret_manager_secret.repo_password_secret, + google_secret_manager_secret_version.repo_password_secret_version, + ] +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/metadata.yaml new file mode 100644 index 0000000000..6b68c98a54 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - secretmanager.googleapis.com + - artifactregistry.googleapis.com + - cloudbuild.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/outputs.tf new file mode 100644 index 0000000000..92b6dbb165 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/outputs.tf @@ -0,0 +1,18 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "registry_url" { + description = "The URL of the created artifact registry." + value = "${var.region}-docker.pkg.dev/${var.project_id}/${var.deployment_name}" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/validation.tf b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/validation.tf new file mode 100644 index 0000000000..a795060fb7 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/validation.tf @@ -0,0 +1,49 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "terraform_data" "input_validation" { + lifecycle { + precondition { + condition = ( + var.repo_password == null || + (var.use_upstream_credentials && var.repo_mode == "REMOTE_REPOSITORY") + ) + error_message = "repo_password may be set only when repo_mode=REMOTE_REPOSITORY and use_upstream_credentials=true." + } + + precondition { + condition = ( + !var.use_upstream_credentials || + var.repo_mode == "REMOTE_REPOSITORY" + ) + error_message = "use_upstream_credentials is allowed only when repo_mode is REMOTE_REPOSITORY." + } + + precondition { + condition = ( + var.repo_mode != "REMOTE_REPOSITORY" || + (var.repo_public_repository != null || var.repo_mirror_url != null) + ) + error_message = "For a REMOTE_REPOSITORY you must set repo_public_repository or repo_mirror_url." + } + + precondition { + condition = ( + !contains(["APT", "YUM"], var.format) || + (var.repository_base != null && var.repository_path != null) + ) + error_message = "APT/YUM formats require repository_base and repository_path." + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/variables.tf new file mode 100644 index 0000000000..9a4eecb921 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/variables.tf @@ -0,0 +1,122 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "Project ID where the artifact registry and secret are created." + type = string +} + +variable "region" { + description = "Region for the artifact registry." + type = string +} + +variable "deployment_name" { + description = "The name of the current deployment." + type = string +} + +variable "labels" { + description = "Labels to add to the artifact registry. Key-value pairs." + type = map(string) + default = {} +} + +variable "repo_password" { + description = "Optional password/API key. If null, one will be randomly generated." + type = string + default = null +} + +variable "user_managed_replication" { + description = <<-DOC + (Optional) A list of objects to enable user-managed replication. + Each object can have: + location = string + kms_key_name = optional(string) + If empty, auto replication is used. + DOC + type = list(object({ + location = string + kms_key_name = optional(string) + })) + default = [] +} + +variable "format" { + description = "Artifact Registry format (e.g., DOCKER)." + type = string + default = "DOCKER" +} + +variable "repo_mode" { + description = "Artifact Registry mode (STANDARD_REPOSITORY, REMOTE_REPOSITORY, etc.)." + type = string + default = "STANDARD_REPOSITORY" + + validation { + condition = can(regex("^(STANDARD_REPOSITORY|REMOTE_REPOSITORY|VIRTUAL_REPOSITORY)$", var.repo_mode)) + error_message = "repo_mode must be one of STANDARD_REPOSITORY, REMOTE_REPOSITORY, or VIRTUAL_REPOSITORY." + } +} + +variable "repo_public_repository" { + description = <<-DOC + For REMOTE_REPOSITORY, name of a known public repo as per the Terraform module + (e.g., DOCKER_HUB) or null for custom repo. + DOC + type = string + default = null + + # To Do: implement validation + # validation { + # condition = ((var.repo_mode != "REMOTE_REPOSITORY" && var.repo_public_repository == null) || (var.repo_mode == "REMOTE_REPOSITORY" && (var.repo_public_repository != null || var.repo_mirror_url != null))) + # error_message = "If repo_mode is REMOTE_REPOSITORY, you must set either repo_public_repository or repo_mirror_url. Otherwise, leave them null." + # } +} + +variable "repo_mirror_url" { + description = "For REMOTE_REPOSITORY, URL for a custom or common mirror." + type = string + default = null +} + +variable "use_upstream_credentials" { + description = <<-DOC + Configure Service Account to use upstream credentials for REMOTE_REPOSITORY: + If true, a username/password is used for the REMOTE_REPOSITORY mirror. + If false (or if repo_password == null), no password is created at all. + Note: Blueprint credentials will be stored in Secrets Manager. + DOC + type = bool + default = false +} + +variable "repo_username" { + description = "Username for external repository." + type = string + default = null +} + +variable "repository_base" { + description = "For APT/YUM public repos, repository_base (e.g., 'DEBIAN', 'UBUNTU')." + type = string + default = null +} + +variable "repository_path" { + description = "For APT/YUM public repos, repository_path (e.g., 'debian/dists/buster')." + type = string + default = null +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/versions.tf new file mode 100644 index 0000000000..392a7131d2 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/README.md new file mode 100644 index 0000000000..23bf87398a --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/README.md @@ -0,0 +1,76 @@ +## Description + +Creates a BigQuery dataset. + +Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +## Usage +This is a simple usage. + +```yaml + - id: bq-dataset + source: community/modules/database/bigquery-dataset + settings: + dataset_id: my_dataset +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_bigquery_dataset.pbsb](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_dataset) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [dataset\_id](#input\_dataset\_id) | The name of the dataset to be created | `string` | `null` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the dataset. Key-value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [dataset\_id](#output\_dataset\_id) | Name of the dataset that was created. | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/main.tf new file mode 100644 index 0000000000..1a9c4bba60 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/main.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "bigquery-dataset", ghpc_role = "database" }) +} +locals { + dataset_id = var.dataset_id != null ? var.dataset_id : replace("${var.deployment_name}_dataset_${random_id.resource_name_suffix.hex}", "-", "_") +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_bigquery_dataset" "pbsb" { + dataset_id = local.dataset_id + project = var.project_id + labels = local.labels +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml new file mode 100644 index 0000000000..87ff9357e4 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - bigquery.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf new file mode 100644 index 0000000000..9cd8e5df31 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "dataset_id" { + description = "Name of the dataset that was created." + value = google_bigquery_dataset.pbsb.dataset_id +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/variables.tf new file mode 100644 index 0000000000..90c229af6b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/variables.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "dataset_id" { + description = "The name of the dataset to be created" + type = string + default = null +} + +variable "labels" { + description = "Labels to add to the dataset. Key-value pairs." + type = map(string) +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/versions.tf new file mode 100644 index 0000000000..12ddbe842d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/README.md new file mode 100644 index 0000000000..ef67cfef01 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/README.md @@ -0,0 +1,87 @@ +## Description + +Creates a BigQuery table with a specified schema. + +Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +## Usage + +```yaml +id: bq-table + source: community/modules/database/bigquery-table + use: [bq-dataset] + settings: + table_schema: + ' + [ + { + "name": "id", "type": "STRING" + } + ] + ' +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_bigquery_table.pbsb](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_table) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [dataset\_id](#input\_dataset\_id) | Dataset name to be used to create the new BQ Table | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the tables. Key-value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [table\_id](#input\_table\_id) | Table name to be used to create the new BQ Table | `string` | `null` | no | +| [table\_schema](#input\_table\_schema) | Schema used to create the new BQ Table | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [dataset\_id](#output\_dataset\_id) | ID of BQ dataset | +| [table\_id](#output\_table\_id) | ID of created BQ table | +| [table\_name](#output\_table\_name) | Name of created BQ table | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/main.tf new file mode 100644 index 0000000000..73f3923e00 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/main.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "bigquery-table", ghpc_role = "database" }) +} + +locals { + table_id = var.table_id != null ? var.table_id : replace("${var.deployment_name}_table_${random_id.resource_name_suffix.hex}", "-", "_") +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_bigquery_table" "pbsb" { + deletion_protection = false + project = var.project_id + table_id = local.table_id + dataset_id = var.dataset_id + schema = var.table_schema + labels = local.labels +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/metadata.yaml new file mode 100644 index 0000000000..87ff9357e4 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - bigquery.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/outputs.tf new file mode 100644 index 0000000000..4220ec1390 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/outputs.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "table_name" { + description = "Name of created BQ table" + value = google_bigquery_table.pbsb.friendly_name +} +output "table_id" { + description = "ID of created BQ table" + value = google_bigquery_table.pbsb.table_id +} +output "dataset_id" { + description = "ID of BQ dataset" + value = google_bigquery_table.pbsb.dataset_id +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/variables.tf new file mode 100644 index 0000000000..ec474b4e64 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/variables.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "labels" { + description = "Labels to add to the tables. Key-value pairs." + type = map(string) +} + +variable "table_id" { + description = "Table name to be used to create the new BQ Table" + type = string + default = null +} + +variable "dataset_id" { + description = "Dataset name to be used to create the new BQ Table" + type = string +} + +variable "table_schema" { + description = "Schema used to create the new BQ Table" + type = string +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/versions.tf new file mode 100644 index 0000000000..12ddbe842d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md new file mode 100644 index 0000000000..08364c175b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md @@ -0,0 +1,107 @@ +## Description + +terraform-google-sql makes it easy to create a Google CloudSQL instance and +implement high availability settings. This module is meant for use with +Terraform 0.13+ and tested using Terraform 1.0+. + +The cloudsql created here is used to integrate with the slurm cluster to enable +accounting data storage. + +### Example + +```yaml +- id: cloudsql + source: community/modules/database/slurm-cloudsql-federation + use: [network] + settings: + sql_instance_name: slurm-sql6-demo + tier: "db-f1-micro" +``` + +This creates a cloud sql instance, including a database, user that would allow +the slurm cluster to use as an external DB. In addition, it will allow BigQuery +to run federated query through it. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.0 | +| [google](#requirement\_google) | >= 3.83 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_bigquery_connection.connection](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_connection) | resource | +| [google_compute_address.psc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | +| [google_compute_forwarding_rule.psc_consumer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_forwarding_rule) | resource | +| [google_sql_database.database](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database) | resource | +| [google_sql_database_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database_instance) | resource | +| [google_sql_user.users](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_user) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [random_password.password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [authorized\_networks](#input\_authorized\_networks) | IP address ranges as authorized networks of the Cloud SQL for MySQL instances | `list(string)` | `[]` | no | +| [data\_cache\_enabled](#input\_data\_cache\_enabled) | Whether data cache is enabled for the instance. Can be used with ENTERPRISE\_PLUS edition. | `bool` | `false` | no | +| [database\_flags](#input\_database\_flags) | Database flags to set on instance. | `map(string)` | `{}` | no | +| [database\_version](#input\_database\_version) | The version of the database to be created. | `string` | `"MYSQL_8_0"` | no | +| [deletion\_protection](#input\_deletion\_protection) | Whether or not to allow Terraform to destroy the instance. | `string` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [disk\_autoresize](#input\_disk\_autoresize) | Set to false to disable automatic disk grow. | `bool` | `true` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of the database disk in GiB. | `number` | `null` | no | +| [edition](#input\_edition) | value | `string` | `"ENTERPRISE"` | no | +| [enable\_backups](#input\_enable\_backups) | Set true to enable backups | `bool` | `false` | no | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is going to be created in.:
`projects//global/networks/`" | `string` | n/a | yes | +| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection, used only as dependency for Cloud SQL creation. | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [query\_insights](#input\_query\_insights) | Query insights configuration. | 
object({
    enabled                 = optional(bool, false)
    query_plans_per_minute  = optional(number)
    query_string_length     = optional(number)
    record_application_tags = optional(bool)
    record_client_address   = optional(bool)
  })
| `{}` | no | +| [region](#input\_region) | The region where SQL instance will be configured | `string` | n/a | yes | +| [sql\_instance\_name](#input\_sql\_instance\_name) | name given to the sql instance for ease of identificaion | `string` | n/a | yes | +| [sql\_password](#input\_sql\_password) | Password for the SQL database. | `any` | `null` | no | +| [sql\_username](#input\_sql\_username) | Username for the SQL database | `string` | `"slurm"` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Self link of the network where Cloud SQL instance PSC endpoint will be created | `string` | `null` | no | +| [tier](#input\_tier) | The machine type to use for the SQL instance | `string` | n/a | yes | +| [use\_psc\_connection](#input\_use\_psc\_connection) | Create Private Service Connection instead of using Private Service Access peering | `bool` | `false` | no | +| [user\_managed\_replication](#input\_user\_managed\_replication) | Replication parameters that will be used for defined secrets | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cloudsql](#output\_cloudsql) | Describes the cloudsql instance. | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf new file mode 100644 index 0000000000..9b518a1b5f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf @@ -0,0 +1,165 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "slurm-cloudsql-federation", ghpc_role = "database" }) +} + +locals { + user_managed_replication = var.user_managed_replication +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "random_password" "password" { + length = 12 + special = false +} + +locals { + sql_instance_name = var.sql_instance_name == null ? "${var.deployment_name}-sql-${random_id.resource_name_suffix.hex}" : var.sql_instance_name + sql_password = var.sql_password == null ? random_password.password.result : var.sql_password +} + + +resource "google_sql_database_instance" "instance" { + project = var.project_id + depends_on = [var.private_vpc_connection_peering] + name = local.sql_instance_name + region = var.region + deletion_protection = var.deletion_protection + database_version = var.database_version + + settings { + disk_size = var.disk_size_gb + disk_autoresize = var.disk_autoresize + edition = var.edition + tier = var.tier + user_labels = local.labels + + dynamic "data_cache_config" { + for_each = var.edition == "ENTERPRISE_PLUS" ? [""] : [] + content { + data_cache_enabled = var.data_cache_enabled + } + } + + dynamic "database_flags" { + for_each = var.database_flags + content { + name = database_flags.key + value = database_flags.value + } + } + + insights_config { + query_insights_enabled = var.query_insights.enabled + query_plans_per_minute = var.query_insights.query_plans_per_minute + query_string_length = var.query_insights.query_string_length + record_application_tags = var.query_insights.record_application_tags + record_client_address = var.query_insights.record_client_address + } + + ip_configuration { + ipv4_enabled = false + private_network = var.use_psc_connection ? null : var.network_id + enable_private_path_for_google_cloud_services = true + + dynamic "authorized_networks" { + for_each = var.use_psc_connection ? [] : var.authorized_networks + iterator = ip_range + + content { + value = ip_range.value + } + } + dynamic "psc_config" { + for_each = var.use_psc_connection ? [""] : [] + content { + psc_enabled = true + allowed_consumer_projects = [var.project_id] + } + } + } + + backup_configuration { + enabled = var.enable_backups + # to allow easy switching between ENTERPRISE and ENTERPRISE_PLUS + transaction_log_retention_days = 7 + } + } + lifecycle { + precondition { + condition = var.disk_autoresize && var.disk_size_gb == null || !var.disk_autoresize + error_message = "If setting disk_size_gb set disk_autorize to false to prevent re-provisioning of the instance after disk auto-expansion." + } + } +} + + + +resource "google_compute_address" "psc" { + count = var.use_psc_connection ? 1 : 0 + project = var.project_id + name = local.sql_instance_name + address_type = "INTERNAL" + region = var.region + subnetwork = var.subnetwork_self_link + labels = local.labels +} + +resource "google_compute_forwarding_rule" "psc_consumer" { + count = var.use_psc_connection ? 1 : 0 + name = local.sql_instance_name + project = var.project_id + region = var.region + subnetwork = var.subnetwork_self_link + ip_address = google_compute_address.psc[0].self_link + load_balancing_scheme = "" + recreate_closed_psc = true + target = google_sql_database_instance.instance.psc_service_attachment_link +} + +resource "google_sql_database" "database" { + project = var.project_id + name = "slurm_accounting" + instance = google_sql_database_instance.instance.name +} + +resource "google_sql_user" "users" { + project = var.project_id + name = var.sql_username + instance = google_sql_database_instance.instance.name + password = local.sql_password +} + +resource "google_bigquery_connection" "connection" { + provider = google + project = var.project_id + location = var.region + cloud_sql { + instance_id = google_sql_database_instance.instance.connection_name + database = google_sql_database.database.name + type = "MYSQL" + credential { + username = google_sql_user.users.name + password = google_sql_user.users.password + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml new file mode 100644 index 0000000000..fc0cae0859 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - bigqueryconnection.googleapis.com + - sqladmin.googleapis.com + - servicenetworking.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf new file mode 100644 index 0000000000..0d05221cd8 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "cloudsql" { + description = "Describes the cloudsql instance." + sensitive = true + value = { + server_ip = var.use_psc_connection ? google_compute_address.psc[0].address : google_sql_database_instance.instance.ip_address[0].ip_address + user = google_sql_user.users.name + password = google_sql_user.users.password + db_name = google_sql_database.database.name + user_managed_replication = local.user_managed_replication + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf new file mode 100644 index 0000000000..a2f150419e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf @@ -0,0 +1,173 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "authorized_networks" { + description = "IP address ranges as authorized networks of the Cloud SQL for MySQL instances" + type = list(string) + default = [] + nullable = false +} + +variable "database_version" { + description = "The version of the database to be created." + type = string + default = "MYSQL_8_0" + validation { + condition = contains(["MYSQL_5_7", "MYSQL_8_0", "MYSQL_8_4"], var.database_version) + error_message = "The database version must be either MYSQL_5_7, MYSQL_8_0 or MYSQL_8_4." + } +} + +variable "data_cache_enabled" { + description = "Whether data cache is enabled for the instance. Can be used with ENTERPRISE_PLUS edition." + type = bool + default = false +} + +variable "database_flags" { + description = "Database flags to set on instance." + type = map(string) + default = {} + nullable = false +} + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "disk_autoresize" { + description = "Set to false to disable automatic disk grow." + type = bool + default = true +} + +variable "disk_size_gb" { + description = "Size of the database disk in GiB." + type = number + default = null +} + +variable "edition" { + description = "value" + type = string + validation { + condition = contains(["ENTERPRISE", "ENTERPRISE_PLUS"], var.edition) + error_message = "The database edition must be either ENTERPRISE or ENTERPRISE_PLUS" + } + default = "ENTERPRISE" +} + +variable "enable_backups" { + description = "Set true to enable backups" + type = bool + default = false +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "query_insights" { + description = "Query insights configuration." + nullable = false + default = {} + type = object({ + enabled = optional(bool, false) + query_plans_per_minute = optional(number) + query_string_length = optional(number) + record_application_tags = optional(bool) + record_client_address = optional(bool) + }) +} + +variable "region" { + description = "The region where SQL instance will be configured" + type = string +} + +variable "tier" { + description = "The machine type to use for the SQL instance" + type = string +} + +variable "sql_instance_name" { + description = "name given to the sql instance for ease of identificaion" + type = string +} + +variable "deletion_protection" { + description = "Whether or not to allow Terraform to destroy the instance." + type = string + default = false +} + +variable "labels" { + description = "Labels to add to the instances. Key-value pairs." + type = map(string) +} + +variable "sql_username" { + description = "Username for the SQL database" + type = string + default = "slurm" +} + +variable "sql_password" { + description = "Password for the SQL database." + type = any + default = null +} + +variable "network_id" { + description = <<-EOT + The ID of the GCE VPC network to which the instance is going to be created in.: + `projects//global/networks/`" + EOT + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "private_vpc_connection_peering" { + description = "The name of the VPC Network peering connection, used only as dependency for Cloud SQL creation." + type = string + default = null +} + +variable "subnetwork_self_link" { + description = "Self link of the network where Cloud SQL instance PSC endpoint will be created" + type = string + default = null +} + +variable "user_managed_replication" { + type = list(object({ + location = string + kms_key_name = optional(string) + })) + description = "Replication parameters that will be used for defined secrets" + default = [] +} + +variable "use_psc_connection" { + description = "Create Private Service Connection instead of using Private Service Access peering" + type = bool + default = false +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf new file mode 100644 index 0000000000..922e383a69 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.57.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.57.0" + } + + required_version = ">= 0.13.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md new file mode 100644 index 0000000000..d39a58afe1 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md @@ -0,0 +1,158 @@ +> [!WARNING] +> This module is deprecated and will be removed on July 1, 2025. The +> recommended replacement is the +> [GCP Managed Lustre module](../../../../modules/file-system/managed-lustre/README.md) + +## Description +This module creates a DDN EXAScaler Cloud Lustre file system using code based on DDN's +[exascaler-cloud-terraform](https://github.com/DDNStorage/exascaler-cloud-terraform/tree/scripts/2.2.2/gcp) (`scripts/2.2.2` is last release with GCP-specific module). + +More information about the architecture can be found at +[Overview of Lustre and EXAScaler Cloud][architecture]. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). + +> **Warning**: This file system has a license cost as described in the pricing +> section of the [DDN EXAScaler Cloud Marketplace Solution][marketplace]. +> +> **Note**: By default security.public_key is set to `null`, therefore the +> admin user is not created. To ensure the admin user is created, provide a +> public key via the security setting. +> +> **Note**: This module's instances require access to Google APIs and +> therefore, instances must have public IP address or it must be used in a +> subnetwork where [Private Google Access][private-google-access] is enabled. + +[private-google-access]: https://cloud.google.com/vpc/docs/configure-private-google-access +[marketplace]: https://console.developers.google.com/marketplace/product/ddnstorage/exascaler-cloud +[architecture]: https://cloud.google.com/architecture/parallel-file-systems-for-hpc#overview_of_lustre_and_exascaler_cloud + +## Mounting + +To mount the DDN EXAScaler Lustre file system you must first install the DDN +Lustre client and then call the proper `mount` command. + +Both of these steps are automatically handled with the use of the `use` command +in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in +the network storage doc for a complete list of supported modules. +the [hpc-enterprise-slurm.yaml](../../../../examples/hpc-enterprise-slurm.yaml) for an +example of using this module with Slurm. + +If mounting is not automatically handled as described above, the DDN-EXAScaler +module outputs runners that can be used with the startup-script module to +install the client and mount the file system. See the following example: + +```yaml + # This file system has an associated license cost. + # https://console.developers.google.com/marketplace/product/ddnstorage/exascaler-cloud + - id: lustrefs + source: community/modules/file-system/DDN-EXAScaler + use: [network1] + settings: {local_mount: /scratch} + + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: + - $(lustrefs.install_ddn_lustre_client_runner) + - $(lustrefs.mount_runner) + +``` + +See [additional documentation][ddn-install-docs] from DDN EXAScaler. + +[ddn-install-docs]: https://github.com/DDNStorage/exascaler-cloud-terraform/tree/scripts/2.2.2/gcp#install-new-exascaler-cloud-clients +[matrix]: ../../../../docs/network_storage.md#compatibility-matrix + +## Support + +EXAScaler Cloud includes self-help support with access to publicly available +documents and videos. Premium support includes 24x7x365 access to DDN's experts, +along with support community access, automated notifications of updates and +other premium support features. For more information, visit +[EXAscaler Cloud on GCP][exa-gcp]. + +[exa-gcp]: https://console.cloud.google.com/marketplace/product/ddnstorage/exascaler-cloud + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [ddn\_exascaler](#module\_ddn\_exascaler) | github.com/DDNStorage/exascaler-cloud-terraform//gcp | a3355d50deebe45c0556b45bd599059b7c06988d | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [boot](#input\_boot) | Boot disk properties | 
object({
    disk_type   = string
    auto_delete = bool
    script_url  = string
  })
| 
{
  "auto_delete": true,
  "disk_type": "pd-standard",
  "script_url": null
}
| no | +| [cls](#input\_cls) | Compute client properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 0,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-2",
  "public_ip": true
}
| no | +| [clt](#input\_clt) | Compute client target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 0,
  "disk_size": 256,
  "disk_type": "pd-standard"
}
| no | +| [fsname](#input\_fsname) | EXAScaler filesystem name, only alphanumeric characters are allowed, and the value must be 1-8 characters long | `string` | `"exacloud"` | no | +| [image](#input\_image) | DEPRECATED: Source image properties | `any` | `null` | no | +| [instance\_image](#input\_instance\_image) | Source image properties

Expected Fields:
name: Unavailable with this module.
family: The image family to use.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "exascaler-cloud-6-2-rocky-linux-8-optimized-gcp",
  "project": "ddn-public"
}
| no | +| [labels](#input\_labels) | Labels to add to EXAScaler Cloud deployment. Key-value pairs. | `map(string)` | `{}` | no | +| [local\_mount](#input\_local\_mount) | Mountpoint (at the client instances) for this EXAScaler system | `string` | `"/shared"` | no | +| [mds](#input\_mds) | Metadata server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 1,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-32",
  "public_ip": true
}
| no | +| [mdt](#input\_mdt) | Metadata target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 3500,
  "disk_type": "pd-ssd"
}
| no | +| [mgs](#input\_mgs) | Management server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 1,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-32",
  "public_ip": true
}
| no | +| [mgt](#input\_mgt) | Management target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 128,
  "disk_type": "pd-standard"
}
| no | +| [mnt](#input\_mnt) | Monitoring target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 128,
  "disk_type": "pd-standard"
}
| no | +| [network\_properties](#input\_network\_properties) | Network options. 'network\_self\_link' or 'network\_properties' must be provided. | 
object({
    routing = string
    tier    = string
    id      = string
    auto    = bool
    mtu     = number
    new     = bool
    nat     = bool
  })
| `null` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self-link of the VPC network to where the system is connected. Ignored if 'network\_properties' is provided. 'network\_self\_link' or 'network\_properties' must be provided. | `string` | `null` | no | +| [oss](#input\_oss) | Object Storage server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 3,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-16",
  "public_ip": true
}
| no | +| [ost](#input\_ost) | Object Storage target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 3500,
  "disk_type": "pd-ssd"
}
| no | +| [prefix](#input\_prefix) | EXAScaler Cloud deployment prefix (`null` defaults to 'exascaler-cloud') | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Compute Platform project that will host the EXAScaler filesystem | `string` | n/a | yes | +| [security](#input\_security) | Security options | 
object({
    admin              = string
    public_key         = string
    block_project_keys = bool
    enable_os_login    = bool
    enable_local       = bool
    enable_ssh         = bool
    enable_http        = bool
    ssh_source_ranges  = list(string)
    http_source_ranges = list(string)
  })
| 
{
  "admin": "stack",
  "block_project_keys": false,
  "enable_http": false,
  "enable_local": false,
  "enable_os_login": true,
  "enable_ssh": false,
  "http_source_ranges": [
    "0.0.0.0/0"
  ],
  "public_key": null,
  "ssh_source_ranges": [
    "0.0.0.0/0"
  ]
}
| no | +| [service\_account](#input\_service\_account) | Service account name used by deploy application | 
object({
    new   = bool
    email = string
  })
| 
{
  "email": null,
  "new": false
}
| no | +| [subnetwork\_address](#input\_subnetwork\_address) | The IP range of internal addresses for the subnetwork. Ignored if 'subnetwork\_properties' is provided. | `string` | `null` | no | +| [subnetwork\_properties](#input\_subnetwork\_properties) | Subnetwork properties. 'subnetwork\_self\_link' or 'subnetwork\_properties' must be provided. | 
object({
    address = string
    private = bool
    id      = string
    new     = bool
  })
| `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self-link of the VPC subnetwork to where the system is connected. Ignored if 'subnetwork\_properties' is provided. 'subnetwork\_self\_link' or 'subnetwork\_properties' must be provided. | `string` | `null` | no | +| [waiter](#input\_waiter) | Waiter to check progress and result for deployment. | `string` | `null` | no | +| [zone](#input\_zone) | Compute Platform zone where the servers will be located | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [client\_config\_script](#output\_client\_config\_script) | Script that will install DDN EXAScaler lustre client. The machine running this script must be on the same network & subnet as the EXAScaler. | +| [http\_console](#output\_http\_console) | HTTP address to access the system web console. | +| [install\_ddn\_lustre\_client\_runner](#output\_install\_ddn\_lustre\_client\_runner) | Runner that encapsulates the `client_config_script` output on this module. | +| [mount\_command](#output\_mount\_command) | Command to mount the file system. `client_config_script` must be run first. | +| [mount\_runner](#output\_mount\_runner) | Runner to mount the DDN EXAScaler Lustre file system | +| [network\_storage](#output\_network\_storage) | Describes a EXAScaler system to be mounted by other systems. | +| [private\_addresses](#output\_private\_addresses) | Private IP addresses for all instances. | +| [ssh\_console](#output\_ssh\_console) | Instructions to ssh into the instances. | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf new file mode 100644 index 0000000000..6a2fc4b702 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf @@ -0,0 +1,72 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# WARNING +# This module is deprecated and will be removed on July 1, 2025 +# The recommended replacement is the Managed Lustre module +# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "ddn-exascaler", ghpc_role = "file-system" }) +} + +locals { + + network_id = var.network_self_link != null ? regex("https://www.googleapis.com/compute/v\\d/(.*)", var.network_self_link)[0] : null + named_net = { + routing = "REGIONAL" + tier = "STANDARD" + id = local.network_id + auto = false + mtu = 1500 + new = false + nat = false + } + + subnetwork_id = var.subnetwork_self_link != null ? regex("https://www.googleapis.com/compute/v\\d/(.*)", var.subnetwork_self_link)[0] : null + named_subnet = { + address = var.subnetwork_address + private = true + id = local.subnetwork_id + new = false + } +} + +module "ddn_exascaler" { + source = "github.com/DDNStorage/exascaler-cloud-terraform//gcp?ref=a3355d50deebe45c0556b45bd599059b7c06988d" + fsname = var.fsname + zone = var.zone + project = var.project_id + prefix = var.prefix + labels = local.labels + security = var.security + service_account = var.service_account + waiter = var.waiter + network = var.network_properties == null ? local.named_net : var.network_properties + subnetwork = var.subnetwork_properties == null ? local.named_subnet : var.subnetwork_properties + boot = var.boot + image = var.instance_image + mgs = var.mgs + mgt = var.mgt + mnt = var.mnt + mds = var.mds + mdt = var.mdt + oss = var.oss + ost = var.ost + cls = var.cls + clt = var.clt +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml new file mode 100644 index 0000000000..b995bd4358 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml @@ -0,0 +1,22 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - deploymentmanager.googleapis.com + - iam.googleapis.com + - runtimeconfig.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf new file mode 100644 index 0000000000..2e9ae732ae --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf @@ -0,0 +1,90 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# WARNING +# This module is deprecated and will be removed on July 1, 2025 +# The recommended replacement is the Managed Lustre module +# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre + +output "private_addresses" { + description = "Private IP addresses for all instances." + value = module.ddn_exascaler.private_addresses +} + +output "ssh_console" { + description = "Instructions to ssh into the instances." + value = module.ddn_exascaler.ssh_console +} + +output "client_config_script" { + description = "Script that will install DDN EXAScaler lustre client. The machine running this script must be on the same network & subnet as the EXAScaler." + value = module.ddn_exascaler.client_config +} + +output "install_ddn_lustre_client_runner" { + description = "Runner that encapsulates the `client_config_script` output on this module." + value = local.client_install_runner +} + +locals { + client_install_runner = { + "type" = "shell" + "content" = module.ddn_exascaler.client_config + "destination" = "install_ddn_lustre_client.sh" + } + + # Mount command provided by DDN does not support custom local mount + split_mount_cmd = split(" ", module.ddn_exascaler.mount_command) + split_mount_cmd_wo_mountpoint = slice(local.split_mount_cmd, 0, length(local.split_mount_cmd) - 1) + mount_cmd = "${join(" ", local.split_mount_cmd_wo_mountpoint)} ${var.local_mount}" + mount_cmd_w_mkdir = "mkdir -p ${var.local_mount} && ${local.mount_cmd}" + mount_runner = { + "type" = "shell" + "content" = local.mount_cmd_w_mkdir + "destination" = "mount-ddn-lustre.sh" + } +} + +output "mount_command" { + description = "Command to mount the file system. `client_config_script` must be run first." + value = local.mount_cmd_w_mkdir +} + +output "mount_runner" { + description = "Runner to mount the DDN EXAScaler Lustre file system" + value = local.mount_runner +} + +output "http_console" { + description = "HTTP address to access the system web console." + value = module.ddn_exascaler.http_console +} + +output "network_storage" { + description = "Describes a EXAScaler system to be mounted by other systems." + value = { + server_ip = split(":", split(" ", module.ddn_exascaler.mount_command)[3])[0] + remote_mount = length(regexall("^/.*", var.fsname)) > 0 ? var.fsname : format("/%s", var.fsname) + local_mount = var.local_mount != null ? var.local_mount : format("/mnt/%s", var.fsname) + fs_type = "lustre" + mount_options = "" + client_install_runner = local.client_install_runner + mount_runner = local.mount_runner + } + depends_on = [ + module.ddn_exascaler + ] +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf new file mode 100644 index 0000000000..68bcc8a8ba --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf @@ -0,0 +1,502 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# WARNING +# This module is deprecated and will be removed on July 1, 2025 +# The recommended replacement is the Managed Lustre module +# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre + +# EXAScaler filesystem name +# only alphanumeric characters are allowed, +# and the value must be 1-8 characters long +variable "fsname" { + description = "EXAScaler filesystem name, only alphanumeric characters are allowed, and the value must be 1-8 characters long" + type = string + default = "exacloud" +} + +# Project ID to manage resources +# https://cloud.google.com/resource-manager/docs/creating-managing-projects +variable "project_id" { + description = "Compute Platform project that will host the EXAScaler filesystem" + type = string +} + +# Zone name to manage resources +# https://cloud.google.com/compute/docs/regions-zones +variable "zone" { + description = "Compute Platform zone where the servers will be located" + type = string +} + +# Service account name used by deploy application +# https://cloud.google.com/iam/docs/service-accounts +# new: create a new custom service account or use an existing one: true or false +# email: existing service account email address, will be using if new is false +# set email = null to use the default compute service account +variable "service_account" { + description = "Service account name used by deploy application" + type = object({ + new = bool + email = string + }) + default = { + new = false + email = null + } +} + +# Waiter to check progress and result for deployment. +# To use Google Deployment Manager: +# waiter = "deploymentmanager" +# To use generic Google Cloud SDK command line: +# waiter = "sdk" +# If you donā€™t want to wait until the deployment is complete: +# waiter = null +# https://cloud.google.com/deployment-manager/runtime-configurator/creating-a-waiter +variable "waiter" { + description = "Waiter to check progress and result for deployment." + type = string + default = null +} + +# Security options +# admin: optional user name for remote SSH access +# Set admin = null to disable creation admin user +# public_key: path to the SSH public key on the local host +# Set public_key = null to disable creation admin user +# block_project_keys: true or false +# Block project-wide public SSH keys if you want to restrict +# deployment to only user with deployment-level public SSH key. +# https://cloud.google.com/compute/docs/instances/adding-removing-ssh-keys +# enable_os_login: true or false +# Enable or disable OS Login feature. +# Please note, enabling this option disables other security options: +# admin, public_key and block_project_keys. +# https://cloud.google.com/compute/docs/instances/managing-instance-access#enable_oslogin +# enable_local: true or false, enable or disable firewall rules for local access +# enable_ssh: true or false, enable or disable remote SSH access +# ssh_source_ranges: source IP ranges for remote SSH access in CIDR notation +# enable_http: true or false, enable or disable remote HTTP access +# http_source_ranges: source IP ranges for remote HTTP access in CIDR notation +variable "security" { + description = "Security options" + type = object({ + admin = string + public_key = string + block_project_keys = bool + enable_os_login = bool + enable_local = bool + enable_ssh = bool + enable_http = bool + ssh_source_ranges = list(string) + http_source_ranges = list(string) + }) + + default = { + admin = "stack" + public_key = null + block_project_keys = false + enable_os_login = true + enable_local = false + enable_ssh = false + enable_http = false + ssh_source_ranges = [ + "0.0.0.0/0" + ] + http_source_ranges = [ + "0.0.0.0/0" + ] + } +} + +variable "network_self_link" { + description = "The self-link of the VPC network to where the system is connected. Ignored if 'network_properties' is provided. 'network_self_link' or 'network_properties' must be provided." + type = string + default = null +} + +# Network properties +# https://cloud.google.com/vpc/docs/vpc +# routing: network-wide routing mode: REGIONAL or GLOBAL +# tier: networking tier for VM interfaces: STANDARD or PREMIUM +# id: existing network id, will be using if new is false +# auto: create subnets in each region automatically: false or true +# mtu: maximum transmission unit in bytes: 1460 - 1500 +# new: create a new network or use an existing one: true or false +# nat: allow instances without external IP to communicate with the outside world: true or false +variable "network_properties" { + description = "Network options. 'network_self_link' or 'network_properties' must be provided." + type = object({ + routing = string + tier = string + id = string + auto = bool + mtu = number + new = bool + nat = bool + }) + + default = null +} + +variable "subnetwork_self_link" { + description = "The self-link of the VPC subnetwork to where the system is connected. Ignored if 'subnetwork_properties' is provided. 'subnetwork_self_link' or 'subnetwork_properties' must be provided." + type = string + default = null +} + +variable "subnetwork_address" { + description = "The IP range of internal addresses for the subnetwork. Ignored if 'subnetwork_properties' is provided." + type = string + default = null +} + +# Subnetwork properties +# https://cloud.google.com/vpc/docs/vpc +# address: IP range of internal addresses for a new subnetwork +# private: when enabled VMs in this subnetwork without external +# IP addresses can access Google APIs and services by using +# Private Google Access: true or false +# https://cloud.google.com/vpc/docs/private-access-options +# id: existing subnetwork id, will be using if new is false +# new: create a new subnetwork or use an existing one: true or false +variable "subnetwork_properties" { + description = "Subnetwork properties. 'subnetwork_self_link' or 'subnetwork_properties' must be provided." + type = object({ + address = string + private = bool + id = string + new = bool + }) + default = null +} +# Boot disk properties +# disk_type: pd-standard, pd-ssd or pd-balanced +# auto_delete: true or false +# whether the disk will be auto-deleted when the instance is deleted +variable "boot" { + description = "Boot disk properties" + type = object({ + disk_type = string + auto_delete = bool + script_url = string + }) + default = { + disk_type = "pd-standard" + auto_delete = true + script_url = null + } +} + +# Source image properties +# project: project name +# family: image family name +# name: !!DEPRECATED!! - image name +# tflint-ignore: terraform_unused_declarations +variable "image" { + description = "DEPRECATED: Source image properties" + type = any + # Omitting type checking so validation can provide more useful error message + # type = object({ + # project = string + # family = string + # }) + default = null + + validation { + condition = var.image == null + error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." + } +} + +variable "instance_image" { + description = <<-EOD + Source image properties + + Expected Fields: + name: Unavailable with this module. + family: The image family to use. + project: The project where the image is hosted. + EOD + type = map(string) + default = { + project = "ddn-public" + family = "exascaler-cloud-6-2-rocky-linux-8-optimized-gcp" + } + + validation { + condition = !can(coalesce(var.instance_image.name)) + error_message = "In var.instance_image, the \"name\" field is not used, please use the \"family\" setting." + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, the \"family\" field must be a string set to the image family." + } +} + +# Management server properties +# node_type: type of management server +# https://cloud.google.com/compute/docs/machine-types +# node_cpu: CPU family +# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform +# nic_type: type of network connectivity, GVNIC or VIRTIO_NET +# https://cloud.google.com/compute/docs/networking/using-gvnic +# public_ip: assign an external IP address, true or false +# node_count: number of management servers +variable "mgs" { + description = "Management server properties" + type = object({ + node_type = string + node_cpu = string + nic_type = string + node_count = number + public_ip = bool + }) + default = { + node_type = "n2-standard-32" + node_cpu = "Intel Cascade Lake" + nic_type = "GVNIC" + public_ip = true + node_count = 1 + } +} + +# Management target properties +# https://cloud.google.com/compute/docs/disks +# disk_bus: type of management target interface, SCSI or NVME (NVME is for scratch disks only) +# disk_type: type of management target, pd-standard, pd-ssd, pd-balanced or scratch +# disk_size: size of management target in GB (scratch disk size must be exactly 375) +# disk_count: number of management targets +# disk_raid: create striped management target, true or false +variable "mgt" { + description = "Management target properties" + type = object({ + disk_bus = string + disk_type = string + disk_size = number + disk_count = number + disk_raid = bool + }) + default = { + disk_bus = "SCSI" + disk_type = "pd-standard" + disk_size = 128 + disk_count = 1 + disk_raid = false + } +} + + +# Monitoring target properties +# https://cloud.google.com/compute/docs/disks +# disk_bus: type of monitoring target interface, SCSI or NVME (NVME is for scratch disks only) +# disk_type: type of monitoring target, pd-standard, pd-ssd, pd-balanced or scratch +# disk_size: size of monitoring target in GB (scratch disk size must be exactly 375) +# disk_count: number of monitoring targets +# disk_raid: create striped monitoring target, true or false +variable "mnt" { + description = "Monitoring target properties" + type = object({ + disk_bus = string + disk_type = string + disk_size = number + disk_count = number + disk_raid = bool + }) + default = { + disk_bus = "SCSI" + disk_type = "pd-standard" + disk_size = 128 + disk_count = 1 + disk_raid = false + } +} + +# Metadata server properties +# node_type: type of metadata server +# https://cloud.google.com/compute/docs/machine-types +# node_cpu: CPU family +# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform +# nic_type: type of network connectivity, GVNIC or VIRTIO_NET +# https://cloud.google.com/compute/docs/networking/using-gvnic +# public_ip: assign an external IP address, true or false +# node_count: number of metadata servers +variable "mds" { + description = "Metadata server properties" + type = object({ + node_type = string + node_cpu = string + nic_type = string + node_count = number + public_ip = bool + }) + default = { + node_type = "n2-standard-32" + node_cpu = "Intel Cascade Lake" + nic_type = "GVNIC" + public_ip = true + node_count = 1 + } +} + +# Metadata target properties +# https://cloud.google.com/compute/docs/disks +# disk_bus: type of metadata target interface, SCSI or NVME (NVME is for scratch disks only) +# disk_type: type of metadata target, pd-standard, pd-ssd, pd-balanced or scratch +# disk_size: size of metadata target in GB (scratch disk size must be exactly 375) +# disk_count: number of metadata targets +# disk_raid: create striped metadata target, true or false +variable "mdt" { + description = "Metadata target properties" + type = object({ + disk_bus = string + disk_type = string + disk_size = number + disk_count = number + disk_raid = bool + }) + default = { + disk_bus = "SCSI" + disk_type = "pd-ssd" + disk_size = 3500 + disk_count = 1 + disk_raid = false + } +} + +# Object Storage server properties +# node_type: type of storage server +# https://cloud.google.com/compute/docs/machine-types +# node_cpu: CPU family +# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform +# nic_type: type of network connectivity, GVNIC or VIRTIO_NET +# https://cloud.google.com/compute/docs/networking/using-gvnic +# public_ip: assign an external IP address, true or false +# node_count: number of storage servers +variable "oss" { + description = "Object Storage server properties" + type = object({ + node_type = string + node_cpu = string + nic_type = string + node_count = number + public_ip = bool + }) + default = { + node_type = "n2-standard-16" + node_cpu = "Intel Cascade Lake" + nic_type = "GVNIC" + public_ip = true + node_count = 3 + } +} + +# Object Storage target properties +# https://cloud.google.com/compute/docs/disks +# disk_bus: type of storage target interface, SCSI or NVME (NVME is for scratch disks only) +# disk_type: type of storage target, pd-standard, pd-ssd, pd-balanced or scratch +# disk_size: size of storage target in GB (scratch disk size must be exactly 375) +# disk_count: number of storage targets +# disk_raid: create striped storage target, true or false +variable "ost" { + description = "Object Storage target properties" + type = object({ + disk_bus = string + disk_type = string + disk_size = number + disk_count = number + disk_raid = bool + }) + default = { + disk_bus = "SCSI" + disk_type = "pd-ssd" + disk_size = 3500 + disk_count = 1 + disk_raid = false + } +} + +# Compute client properties +# node_type: type of compute client +# https://cloud.google.com/compute/docs/machine-types +# node_cpu: CPU family +# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform +# nic_type: type of network connectivity, GVNIC or VIRTIO_NET +# https://cloud.google.com/compute/docs/networking/using-gvnic +# public_ip: assign an external IP address, true or false +# node_count: number of compute clients +variable "cls" { + description = "Compute client properties" + type = object({ + node_type = string + node_cpu = string + nic_type = string + node_count = number + public_ip = bool + }) + default = { + node_type = "n2-standard-2" + node_cpu = "Intel Cascade Lake" + nic_type = "GVNIC" + public_ip = true + node_count = 0 + } +} +# Compute client target properties +# https://cloud.google.com/compute/docs/disks +# disk_bus: type of compute target interface, SCSI or NVME (NVME is for scratch disks only) +# disk_type: type of compute target, pd-standard, pd-ssd, pd-balanced or scratch +# disk_size: size of compute target in GB (scratch disk size must be exactly 375) +# disk_count: number of compute targets +variable "clt" { + description = "Compute client target properties" + type = object({ + disk_bus = string + disk_type = string + disk_size = number + disk_count = number + }) + default = { + disk_bus = "SCSI" + disk_type = "pd-standard" + disk_size = 256 + disk_count = 0 + } +} +variable "local_mount" { + description = "Mountpoint (at the client instances) for this EXAScaler system" + type = string + default = "/shared" +} + +variable "prefix" { + description = "EXAScaler Cloud deployment prefix (`null` defaults to 'exascaler-cloud')" + type = string + default = null +} + +variable "labels" { + description = "Labels to add to EXAScaler Cloud deployment. Key-value pairs." + type = map(string) + default = {} +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf new file mode 100644 index 0000000000..2981b4dd75 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf @@ -0,0 +1,24 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +# WARNING +# This module is deprecated and will be removed on July 1, 2025 +# The recommended replacement is the Managed Lustre module +# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre + +terraform { + required_version = ">= 0.13.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/Intel-DAOS/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/Intel-DAOS/README.md new file mode 100644 index 0000000000..04db0acb8c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/Intel-DAOS/README.md @@ -0,0 +1 @@ +> **_NOTE:_** Cluster Toolkit is dropping support for the external [Google Cloud DAOS](https://github.com/daos-stack/google-cloud-daos/tree/main) repository. The DAOS example blueprints (`hpc-slurm-daos.yaml` and `pfs-daos.yaml`) have been removed from the Cluster Toolkit. We recommend migrating to the first-party [Parallelstore](../../../../modules/file-system/parallelstore/) module for similar functionality. To help with this transition, see the Parallelstore example blueprints ([pfs-parallelstore.yaml](../../../../examples/pfs-parallelstore.yaml) and [ps-slurm.yaml](../../../../examples/ps-slurm.yaml)). If the external [Google Cloud DAOS](https://github.com/daos-stack/google-cloud-daos/tree/main) repository is necessary, we recommend using the last Cluster Toolkit [v1.41.0](https://github.com/GoogleCloudPlatform/cluster-toolkit/releases/tag/v1.41.0). diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/README.md new file mode 100644 index 0000000000..2e3fe7e933 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/README.md @@ -0,0 +1,170 @@ +## Description + +This module creates a [Google Cloud Storage (GCS) bucket](https://cloud.google.com/storage). + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). + +### Example + +The following example will create a bucket named `simulation-results-xxxxxxxx`, +where `xxxxxxxx` is a randomly generated id. + +```yaml + - id: bucket + source: community/modules/file-system/cloud-storage-bucket + settings: + name_prefix: simulation-results + random_suffix: true +``` + +> **_NOTE:_** Use of `random_suffix` may cause the following error when used +> with other modules: +> `value depends on resource attributes that cannot be determined until apply`. +> To resolve this set `random_suffix` to `false` (default). + + + +> **_NOTE:_** Bucket namespace is shared by all users of Google Cloud so it is +> possible to have a bucket name clash with an existing bucket that is not in +> your project. To resolve this try to use a more unique name, or set the +> `random_suffix` variable to `true`. + +## Naming of Bucket + +There are potentially three parts to the bucket name. Each of these parts are +configurable in the blueprint. + +1. A **custom prefix**, provided by the user in the blueprint \ +Provide the custom prefix using the `name_prefix` setting. + +1. The **deployment name**, included by default \ +The deployment name can be excluded by setting `use_deployment_name_in_bucket_name: false`. + +1. A **random id** suffix, excluded by default \ +The random id can be included by setting `random_suffix: true`. + +If none of these are provided (no `name_prefix`, +`use_deployment_name_in_bucket_name: false`, & `random_suffix: false`), then the +bucket name will default to `no-bucket-name-provided`. + +Since bucket namespace is shared by all users of Google Cloud, it is more likely +to experience naming clashes than with other resources. In many cases, adding +the `random_suffix` will resolve the naming clash issue. + +> **Warning**: If a bucket is created with a `random_suffix` and then used as +> the bucket for a startup script in the same deployment group this will cause a +> `not known at apply time` error in terraform. The solution is to either create +> the bucket in a separate deployment group or to remove the random suffix. + +## Mounting + +To mount the Cloud Storage bucket you must first ensure that the GCS Fuse client +has been installed and then call the proper `mount` command. + +Both of these steps are automatically handled with the use of the `use` command +in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in +the network storage doc for a complete list of supported modules. + +If mounting is not automatically handled as described above, the +`cloud-storage-bucket` module outputs runners that can be used with the +`startup-script` module to install the client and mount the file system. See the +following example: + +```yaml + - id: bucket + source: community/modules/file-system/cloud-storage-bucket + settings: {local_mount: /data} + + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: + - $(bucket.client_install_runner) + - $(bucket.mount_runner) +``` + +[matrix]: ../../../../docs/network_storage.md#compatibility-matrix + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [google](#requirement\_google) | >= 3.83 | +| [google-beta](#requirement\_google-beta) | >= 6.9.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [google-beta](#provider\_google-beta) | >= 6.9.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_storage_bucket) | resource | +| [google_storage_bucket_iam_binding.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_binding) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [autoclass](#input\_autoclass) | Configure bucket autoclass setup

The autoclass config supports automatic transitions of objects in the bucket to appropriate storage classes based on each object's access pattern.

The terminal storage class defines that objects in the bucket eventually transition to if they are not read for a certain length of time.
Supported values include: 'NEARLINE', 'ARCHIVE' (Default 'NEARLINE')

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/autoclass | 
object({
    enabled                = optional(bool, false)
    terminal_storage_class = optional(string, null)
  })
| 
{
  "enabled": false
}
| no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment; used as part of name of the GCS bucket. | `string` | n/a | yes | +| [enable\_hierarchical\_namespace](#input\_enable\_hierarchical\_namespace) | If true, enables hierarchical namespace for the bucket. This option must be configured during the initial creation of the bucket. | `bool` | `false` | no | +| [enable\_object\_retention](#input\_enable\_object\_retention) | If true, enables retention policy at per object level for the bucket.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/object-lock | `bool` | `false` | no | +| [enable\_versioning](#input\_enable\_versioning) | If true, enables versioning for the bucket. | `bool` | `false` | no | +| [force\_destroy](#input\_force\_destroy) | If true will destroy bucket with all objects stored within. | `bool` | `false` | no | +| [labels](#input\_labels) | Labels to add to the GCS bucket. Key-value pairs. | `map(string)` | n/a | yes | +| [lifecycle\_rules](#input\_lifecycle\_rules) | List of config to manage data lifecycle rules for the bucket. For more details: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket.html#nested_lifecycle_rule | 
list(object({
    # Object with keys:
    # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass.
    # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule.
    action = object({
      type          = string
      storage_class = optional(string)
    })

    # Object with keys:
    # - age - (Optional) Minimum age of an object in days to satisfy this condition.
    # - send_age_if_zero - (Optional) While set true, num_newer_versions value will be sent in the request even for zero value of the field.
    # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition.
    # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY".
    # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE, STANDARD, DURABLE_REDUCED_AVAILABILITY.
    # - matches_prefix - (Optional) One or more matching name prefixes to satisfy this condition.
    # - matches_suffix - (Optional) One or more matching name suffixes to satisfy this condition.
    # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition.
    # - custom_time_before - (Optional) A date in the RFC 3339 format YYYY-MM-DD. This condition is satisfied when the customTime metadata for the object is set to an earlier date than the date used in this lifecycle condition.
    # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true.
    # - days_since_noncurrent_time - (Optional) Relevant only for versioned objects. Number of days elapsed since the noncurrent timestamp of an object.
    # - noncurrent_time_before - (Optional) Relevant only for versioned objects. The date in RFC 3339 (e.g. 2017-06-13) when the object became nonconcurrent.
    condition = object({
      age                        = optional(number)
      send_age_if_zero           = optional(bool)
      created_before             = optional(string)
      with_state                 = optional(string)
      matches_storage_class      = optional(string)
      matches_prefix             = optional(string)
      matches_suffix             = optional(string)
      num_newer_versions         = optional(number)
      custom_time_before         = optional(string)
      days_since_custom_time     = optional(number)
      days_since_noncurrent_time = optional(number)
      noncurrent_time_before     = optional(string)
    })
  }))
| `[]` | no | +| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | +| [mount\_options](#input\_mount\_options) | Mount options to be put in fstab. Note: `implicit_dirs` makes it easier to work with objects added by other tools, but there is a performance impact. See: [more information](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#implicit-directories) | `string` | `"defaults,_netdev,implicit_dirs"` | no | +| [name\_prefix](#input\_name\_prefix) | Name Prefix. | `string` | `null` | no | +| [project\_id](#input\_project\_id) | ID of project in which GCS bucket will be created. | `string` | n/a | yes | +| [public\_access\_prevention](#input\_public\_access\_prevention) | Bucket public access can be controlled by setting a value of either `inherited` or `enforced`.
When set to `enforced`, public access to the bucket is blocked.
If set to `inherited`, the bucket's public access prevention depends on whether it is subject to the organization policy constraint for public access prevention.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/public-access-prevention | `string` | `null` | no | +| [random\_suffix](#input\_random\_suffix) | If true, a random id will be appended to the suffix of the bucket name. | `bool` | `false` | no | +| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | +| [retention\_policy\_period](#input\_retention\_policy\_period) | If defined, this will configure retention\_policy with retention\_period for the bucket, value must be in between 1 and 3155760000(100 years) seconds.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/bucket-lock | `number` | `null` | no | +| [soft\_delete\_retention\_duration](#input\_soft\_delete\_retention\_duration) | If defined, this will configure soft\_delete\_policy with retention\_duration\_seconds for the bucket, value can be 0 or in between 604800(7 days) and 7776000(90 days).
Setting a 0 duration disables soft delete, meaning any deleted objects will be permanently deleted.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/soft-delete | `number` | `null` | no | +| [storage\_class](#input\_storage\_class) | The storage class of the GCS bucket. | `string` | `"REGIONAL"` | no | +| [uniform\_bucket\_level\_access](#input\_uniform\_bucket\_level\_access) | Allow uniform control access to the bucket. | `bool` | `true` | no | +| [use\_deployment\_name\_in\_bucket\_name](#input\_use\_deployment\_name\_in\_bucket\_name) | If true, the deployment name will be included as part of the bucket name. This helps prevent naming clashes across multiple deployments. | `bool` | `true` | no | +| [viewers](#input\_viewers) | A list of additional accounts that can read packages from this bucket | `set(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [client\_install\_runner](#output\_client\_install\_runner) | Runner that performs client installation needed to use gcs fuse. | +| [gcs\_bucket\_name](#output\_gcs\_bucket\_name) | Bucket name. | +| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | The gsutil bucket path with format of `gs://`. | +| [mount\_runner](#output\_mount\_runner) | Runner that mounts the cloud storage bucket with gcs fuse. | +| [network\_storage](#output\_network\_storage) | Describes a remote network storage to be mounted by fs-tab. | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/main.tf new file mode 100644 index 0000000000..81ba0ca6a9 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/main.tf @@ -0,0 +1,126 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "cloud-storage-bucket", ghpc_role = "file-system" }) +} + +locals { + prefix = var.name_prefix != null ? var.name_prefix : "" + deployment = var.use_deployment_name_in_bucket_name ? var.deployment_name : "" + suffix = var.random_suffix ? random_id.resource_name_suffix.hex : "" + first_dash = (local.prefix != "" && (local.deployment != "" || local.suffix != "")) ? "-" : "" + second_dash = local.deployment != "" && local.suffix != "" ? "-" : "" + composite_name = "${local.prefix}${local.first_dash}${local.deployment}${local.second_dash}${local.suffix}" + name = local.composite_name == "" ? "no-bucket-name-provided" : local.composite_name +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_storage_bucket" "bucket" { + provider = google-beta + project = var.project_id + name = local.name + uniform_bucket_level_access = var.uniform_bucket_level_access + location = var.region + storage_class = var.storage_class + labels = local.labels + force_destroy = var.force_destroy + public_access_prevention = var.public_access_prevention + enable_object_retention = var.enable_object_retention + hierarchical_namespace { + enabled = var.enable_hierarchical_namespace + } + + dynamic "autoclass" { + for_each = var.autoclass.enabled ? [1] : [] + content { + enabled = var.autoclass.enabled + terminal_storage_class = var.autoclass.terminal_storage_class + } + } + + dynamic "soft_delete_policy" { + for_each = var.soft_delete_retention_duration == null ? [] : [1] + content { + retention_duration_seconds = var.soft_delete_retention_duration + } + } + + dynamic "retention_policy" { + for_each = var.retention_policy_period == null ? [] : [1] + content { + retention_period = var.retention_policy_period + } + } + + dynamic "versioning" { + for_each = var.enable_versioning ? [1] : [] + content { + enabled = var.enable_versioning + } + } + + dynamic "lifecycle_rule" { + for_each = var.lifecycle_rules + content { + action { + type = lifecycle_rule.value.action.type + storage_class = lookup(lifecycle_rule.value.action, "storage_class", null) + } + condition { + age = lookup(lifecycle_rule.value.condition, "age", null) + send_age_if_zero = lookup(lifecycle_rule.value.condition, "send_age_if_zero", null) + created_before = lookup(lifecycle_rule.value.condition, "created_before", null) + with_state = lookup(lifecycle_rule.value.condition, "with_state", contains(keys(lifecycle_rule.value.condition), "is_live") ? (lifecycle_rule.value.condition["is_live"] ? "LIVE" : null) : null) + matches_storage_class = lifecycle_rule.value.condition["matches_storage_class"] != null ? split(",", lifecycle_rule.value.condition["matches_storage_class"]) : null + matches_prefix = lifecycle_rule.value.condition["matches_prefix"] != null ? split(",", lifecycle_rule.value.condition["matches_prefix"]) : null + matches_suffix = lifecycle_rule.value.condition["matches_suffix"] != null ? split(",", lifecycle_rule.value.condition["matches_suffix"]) : null + num_newer_versions = lookup(lifecycle_rule.value.condition, "num_newer_versions", null) + custom_time_before = lookup(lifecycle_rule.value.condition, "custom_time_before", null) + days_since_custom_time = lookup(lifecycle_rule.value.condition, "days_since_custom_time", null) + days_since_noncurrent_time = lookup(lifecycle_rule.value.condition, "days_since_noncurrent_time", null) + noncurrent_time_before = lookup(lifecycle_rule.value.condition, "noncurrent_time_before", null) + } + } + } + + lifecycle { + precondition { + condition = !var.autoclass.enabled || !var.enable_hierarchical_namespace + error_message = "Hierarchical namespace is not compatible with Autoclass enabled." + } + + precondition { + condition = !var.enable_hierarchical_namespace || var.uniform_bucket_level_access + error_message = "Hierarchical namespace is not compatible with Uniform bucket level access disabled." + } + + precondition { + condition = !var.enable_versioning || !var.enable_hierarchical_namespace + error_message = "Hierarchical namespace is not compatible with Object versioning enabled." + } + } +} + +resource "google_storage_bucket_iam_binding" "viewers" { + bucket = google_storage_bucket.bucket.name + role = "roles/storage.objectViewer" + members = var.viewers +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/outputs.tf new file mode 100644 index 0000000000..29ddfef2d2 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/outputs.tf @@ -0,0 +1,69 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "network_storage" { + description = "Describes a remote network storage to be mounted by fs-tab." + value = { + remote_mount = local.name + local_mount = var.local_mount + fs_type = "gcsfuse" + mount_options = var.mount_options + server_ip = "" + client_install_runner = local.client_install_runner + mount_runner = local.mount_runner + } +} + +locals { + client_install_runner = { + "type" = "shell" + "content" = file("${path.module}/scripts/install-gcs-fuse.sh") + "destination" = "install-gcsfuse${replace(var.local_mount, "/", "_")}.sh" + } + + mount_runner = { + "type" = "shell" + "destination" = "mount_gcs${replace(var.local_mount, "/", "_")}.sh" + "args" = "\"not-used\" \"${local.name}\" \"${var.local_mount}\" \"gcsfuse\" \"${var.mount_options}\"" + "content" = file("${path.module}/scripts/mount.sh") + } +} + +output "client_install_runner" { + description = "Runner that performs client installation needed to use gcs fuse." + value = local.client_install_runner +} + +output "mount_runner" { + description = "Runner that mounts the cloud storage bucket with gcs fuse." + value = local.mount_runner +} + +output "gcs_bucket_path" { + description = "The gsutil bucket path with format of `gs://`." + # cannot use resource attribute, will cause lookup failure in startup-script + value = "gs://${local.name}" + + # needed to make sure bucket contents are deleted before bucket + depends_on = [ + google_storage_bucket.bucket + ] +} + +output "gcs_bucket_name" { + description = "Bucket name." + value = google_storage_bucket.bucket.name +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh new file mode 100644 index 0000000000..f8a990260b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +if [ ! "$(which gcsfuse)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ]; then + tee /etc/yum.repos.d/gcsfuse.repo >/dev/null </dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/variables.tf new file mode 100644 index 0000000000..9804e4b268 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/variables.tf @@ -0,0 +1,254 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which GCS bucket will be created." + type = string +} + +variable "deployment_name" { + description = "Name of the HPC deployment; used as part of name of the GCS bucket." + type = string +} + +variable "region" { + description = "The region to deploy to" + type = string +} + +variable "labels" { + description = "Labels to add to the GCS bucket. Key-value pairs." + type = map(string) +} + +variable "local_mount" { + description = "The mount point where the contents of the device may be accessed after mounting." + type = string + default = "/mnt" +} + +variable "mount_options" { + description = "Mount options to be put in fstab. Note: `implicit_dirs` makes it easier to work with objects added by other tools, but there is a performance impact. See: [more information](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#implicit-directories)" + type = string + default = "defaults,_netdev,implicit_dirs" +} + +variable "name_prefix" { + description = "Name Prefix." + type = string + default = null +} + +variable "use_deployment_name_in_bucket_name" { + description = "If true, the deployment name will be included as part of the bucket name. This helps prevent naming clashes across multiple deployments." + type = bool + default = true +} + +variable "random_suffix" { + description = "If true, a random id will be appended to the suffix of the bucket name." + type = bool + default = false +} + +variable "force_destroy" { + description = "If true will destroy bucket with all objects stored within." + type = bool + default = false +} + +variable "viewers" { + description = "A list of additional accounts that can read packages from this bucket" + type = set(string) + default = [] + + validation { + error_message = "All bucket viewers must be in IAM style: user:user@example.com, serviceAccount:sa@example.com, or group:group@example.com." + condition = alltrue([ + for viewer in var.viewers : length(regexall("^(user|serviceAccount|group):", viewer)) > 0 + ]) + } +} + +variable "enable_hierarchical_namespace" { + description = "If true, enables hierarchical namespace for the bucket. This option must be configured during the initial creation of the bucket." + type = bool + default = false +} + +variable "uniform_bucket_level_access" { + description = "Allow uniform control access to the bucket." + type = bool + default = true +} + +variable "storage_class" { + description = "The storage class of the GCS bucket." + type = string + default = "REGIONAL" + validation { + condition = contains([ + "STANDARD", + "MULTI_REGIONAL", + "REGIONAL", + "NEARLINE", + "COLDLINE", + "ARCHIVE" + ], var.storage_class) + error_message = "Allowed values for GCS storage_class are 'STANDARD', 'MULTI_REGIONAL', 'REGIONAL', 'NEARLINE', 'COLDLINE', 'ARCHIVE'.\nhttps://cloud.google.com/storage/docs/storage-classes" + } +} + +variable "autoclass" { + description = <<-EOT + Configure bucket autoclass setup + + The autoclass config supports automatic transitions of objects in the bucket to appropriate storage classes based on each object's access pattern. + + The terminal storage class defines that objects in the bucket eventually transition to if they are not read for a certain length of time. + Supported values include: 'NEARLINE', 'ARCHIVE' (Default 'NEARLINE') + + See Cloud documentation for more details: + + https://cloud.google.com/storage/docs/autoclass + EOT + type = object({ + enabled = optional(bool, false) + terminal_storage_class = optional(string, null) + }) + default = { + enabled = false + } + nullable = false + validation { + condition = !can(coalesce(var.autoclass.terminal_storage_class)) || var.autoclass.enabled + error_message = "Cannot set bucket var.autoclass.terminal_storage_class unless var.autoclass.enabled is true" + } +} + +variable "public_access_prevention" { + description = <<-EOT + Bucket public access can be controlled by setting a value of either `inherited` or `enforced`. + When set to `enforced`, public access to the bucket is blocked. + If set to `inherited`, the bucket's public access prevention depends on whether it is subject to the organization policy constraint for public access prevention. + + See Cloud documentation for more details: + + https://cloud.google.com/storage/docs/public-access-prevention + EOT + type = string + default = null + validation { + condition = var.public_access_prevention == null ? true : contains([ + "inherited", + "enforced" + ], var.public_access_prevention) + error_message = "Allowed values for public_access_prevention are 'inherited', 'enforced'.\n" + } +} + +variable "soft_delete_retention_duration" { + description = <<-EOT + If defined, this will configure soft_delete_policy with retention_duration_seconds for the bucket, value can be 0 or in between 604800(7 days) and 7776000(90 days). + Setting a 0 duration disables soft delete, meaning any deleted objects will be permanently deleted. + + See Cloud documentation for more details: + + https://cloud.google.com/storage/docs/soft-delete + EOT + type = number + default = null + validation { + condition = var.soft_delete_retention_duration == null ? true : var.soft_delete_retention_duration == 0 || var.soft_delete_retention_duration >= 604800 && var.soft_delete_retention_duration <= 7776000 + error_message = "var.soft_delete_retention_duration value can be 0 or in between 604800(7 days) and 7776000(90 days)." + } +} + +variable "enable_versioning" { + description = "If true, enables versioning for the bucket." + type = bool + default = false +} + +variable "lifecycle_rules" { + description = "List of config to manage data lifecycle rules for the bucket. For more details: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket.html#nested_lifecycle_rule" + type = list(object({ + # Object with keys: + # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass. + # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule. + action = object({ + type = string + storage_class = optional(string) + }) + + # Object with keys: + # - age - (Optional) Minimum age of an object in days to satisfy this condition. + # - send_age_if_zero - (Optional) While set true, num_newer_versions value will be sent in the request even for zero value of the field. + # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition. + # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY". + # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE, STANDARD, DURABLE_REDUCED_AVAILABILITY. + # - matches_prefix - (Optional) One or more matching name prefixes to satisfy this condition. + # - matches_suffix - (Optional) One or more matching name suffixes to satisfy this condition. + # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition. + # - custom_time_before - (Optional) A date in the RFC 3339 format YYYY-MM-DD. This condition is satisfied when the customTime metadata for the object is set to an earlier date than the date used in this lifecycle condition. + # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true. + # - days_since_noncurrent_time - (Optional) Relevant only for versioned objects. Number of days elapsed since the noncurrent timestamp of an object. + # - noncurrent_time_before - (Optional) Relevant only for versioned objects. The date in RFC 3339 (e.g. 2017-06-13) when the object became nonconcurrent. + condition = object({ + age = optional(number) + send_age_if_zero = optional(bool) + created_before = optional(string) + with_state = optional(string) + matches_storage_class = optional(string) + matches_prefix = optional(string) + matches_suffix = optional(string) + num_newer_versions = optional(number) + custom_time_before = optional(string) + days_since_custom_time = optional(number) + days_since_noncurrent_time = optional(number) + noncurrent_time_before = optional(string) + }) + })) + default = [] +} + +variable "retention_policy_period" { + description = <<-EOT + If defined, this will configure retention_policy with retention_period for the bucket, value must be in between 1 and 3155760000(100 years) seconds. + + See Cloud documentation for more details: + + https://cloud.google.com/storage/docs/bucket-lock + EOT + type = number + default = null + validation { + condition = var.retention_policy_period == null ? true : var.retention_policy_period > 0 && var.retention_policy_period <= 3155760000 + error_message = "var.soft_delete_policy_retention_duration value must be in between 1 and 3155760000(100 years) seconds." + } +} + +variable "enable_object_retention" { + description = <<-EOT + If true, enables retention policy at per object level for the bucket. + + See Cloud documentation for more details: + + https://cloud.google.com/storage/docs/object-lock + EOT + type = bool + default = false +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/versions.tf new file mode 100644 index 0000000000..e80c28ca47 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/versions.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.9.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.57.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.57.0" + } + required_version = ">= 0.14.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/README.md new file mode 100644 index 0000000000..66aaaa46af --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/README.md @@ -0,0 +1,152 @@ +## Description + +This module creates a Network File Sharing (NFS) file system based on a VM +instance and [compute disk][disk]. This file system can share directories and +files with other clients over a network. `nfs-server` can be used by +[vm-instance](../../../../modules/compute/vm-instance/README.md) and SchedMD +community modules that create compute VMs. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). + +If you are using Hyperdisk storage, check the possible disk size, IOPS, and throughput values for each disk type in the [Hyperdisk limits documentation](https://cloud.google.com/compute/docs/disks/hyperdisks#limits-disk). + +> **_WARNING:_** This module has only been tested against the HPC centos7 OS +> disk image (the default). Using other images may work, but have not been +> verified. + +[disk]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk + +### Example + +```yaml +- id: homefs + source: community/modules/file-system/nfs-server + use: [network1] +``` + +This creates a NFS on a virtual machine which allow other VMs to mount the +volume as an external file system. + +> **_NOTE:_** All disks are destroyed along with the instance, during a `gcluster destroy`/`terraform destroy` event. However, you can setup data retention with `create_boot_snapshot_before_destroy` (boot disk) and `create_snapshot_before_destroy` (data disk). + +## Mounting + +To mount the NFS Server you must first ensure that the NFS client has been +installed the and then call the proper `mount` command. + +Both of these steps are automatically handled with the use of the `use` command +in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in +the network storage doc for a complete list of supported modules. +See the [hpc-centos-ss.yaml] test config for an example of using this module +with a `vm-instance` module. + +If mounting is not automatically handled as described above, the `nfs-server` +module outputs runners that can be used with the startup-script module to +install the client and mount the file system. See the following example: + +```yaml + - id: nfs + source: community/modules/file-system/nfs-server + use: [network1] + settings: {local_mounts: [/mnt1]} + + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: + - $(nfs.install_nfs_client_runner) + - $(nfs.mount_runner) + +``` + +[hpc-centos-ss.yaml]: ../../../../tools/validate_configs/test_configs/hpc-centos-ss.yaml +[matrix]: ../../../../docs/network_storage.md#compatibility-matrix + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [google](#requirement\_google) | >= 6.14 | +| [null](#requirement\_null) | >= 3.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.14 | +| [null](#provider\_null) | >= 3.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_disk.attached_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [google_compute_disk.boot_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [google_compute_instance.compute_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance) | resource | +| [null_resource.image](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [google_compute_default_service_account.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_default_service_account) | data source | +| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [auto\_delete\_disk](#input\_auto\_delete\_disk) | DEPRECATED: Whether or not the NFS disk should be auto-deleted | `string` | `null` | no | +| [boot\_disk\_size](#input\_boot\_disk\_size) | Storage size in GB for the boot disk | `number` | `null` | no | +| [boot\_disk\_type](#input\_boot\_disk\_type) | Storage type for the boot disk | `string` | `null` | no | +| [create\_boot\_snapshot\_before\_destroy](#input\_create\_boot\_snapshot\_before\_destroy) | Whether to create a snapshot before destroying the boot disk | `bool` | `false` | no | +| [create\_snapshot\_before\_destroy](#input\_create\_snapshot\_before\_destroy) | Whether to create a snapshot before destroying the NFS data disk | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the NFS instance if no name is specified. | `string` | n/a | yes | +| [disk\_size](#input\_disk\_size) | Storage size in GB for the NFS data disk | `number` | `"100"` | no | +| [image](#input\_image) | DEPRECATED: The VM image used by the NFS server | `string` | `null` | no | +| [instance\_image](#input\_instance\_image) | The VM image used by the NFS server.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | +| [labels](#input\_labels) | Labels to add to the NFS instance. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_mounts](#input\_local\_mounts) | Mountpoint for this NFS compute instance | `list(string)` | 
[
  "/data"
]
| no | +| [machine\_type](#input\_machine\_type) | Type of the VM instance to use | `string` | `"n2d-standard-2"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | +| [name](#input\_name) | The resource name of the instance. | `string` | `null` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the NFS VM. | `string` | `"default"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [provisioned\_iops](#input\_provisioned\_iops) | Provisioned IOPS for the NFS data disk if using Extreme PD or Hyperdisk Balanced/ML/Throughput | `number` | `null` | no | +| [provisioned\_throughput](#input\_provisioned\_throughput) | Provisioned throughput for the NFS data disk if using Hyperdisk Balanced/Extreme | `number` | `null` | no | +| [scopes](#input\_scopes) | Scopes to apply to the controller | `list(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [service\_account](#input\_service\_account) | Service Account for the NFS server | `string` | `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the NFS VM. | `string` | `null` | no | +| [type](#input\_type) | Storage type for the NFS data disk | `string` | `"pd-ssd"` | no | +| [zone](#input\_zone) | The zone name where the NFS instance located in. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | +| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | +| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | +| [network\_storage](#output\_network\_storage) | export of all desired folder directories | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/main.tf new file mode 100644 index 0000000000..a00d2681ba --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/main.tf @@ -0,0 +1,131 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "nfs-server", ghpc_role = "file-system" }) +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +locals { + name = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" + server_ip = google_compute_instance.compute_instance.network_interface[0].network_ip + fs_type = "nfs" + mount_options = "defaults,hard,intr" + install_nfs_client_runners = [for mount in var.local_mounts : + { + "type" = "shell" + "source" = "${path.module}/scripts/install-nfs-client.sh" + "destination" = "install-nfs${replace(mount, "/", "_")}.sh" + } + ] + mount_runners = [for mount in var.local_mounts : + { + "type" = "shell" + "source" = "${path.module}/scripts/mount.sh" + "args" = "\"${local.server_ip}\" \"/exports${mount}\" \"${mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" + "destination" = "mount${replace(mount, "/", "_")}.sh" + } + ] + ansible_mount_runner = { + "type" = "ansible-local" + "source" = "${path.module}/scripts/mount.yaml" + "destination" = "mount.yaml" + } +} + +data "google_compute_default_service_account" "default" {} + +resource "google_compute_disk" "attached_disk" { + project = var.project_id + name = "${local.name}-nfs-instance-disk" + size = var.disk_size + type = var.type + zone = var.zone + labels = local.labels + provisioned_iops = var.provisioned_iops + provisioned_throughput = var.provisioned_throughput + create_snapshot_before_destroy = var.create_snapshot_before_destroy +} + +data "google_compute_image" "compute_image" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = var.instance_image.project +} + +resource "null_resource" "image" { + triggers = { + name = try(var.instance_image.name, null), + family = try(var.instance_image.family, null), + project = var.instance_image.project + } +} + +resource "google_compute_disk" "boot_disk" { + project = var.project_id + + name = "${local.name}-boot-disk" + size = var.boot_disk_size + type = var.boot_disk_type + image = data.google_compute_image.compute_image.self_link + labels = local.labels + zone = var.zone + create_snapshot_before_destroy = var.create_boot_snapshot_before_destroy + + lifecycle { + replace_triggered_by = [null_resource.image] + ignore_changes = [ + image + ] + } +} + +resource "google_compute_instance" "compute_instance" { + project = var.project_id + name = "${local.name}-nfs-instance" + zone = var.zone + machine_type = var.machine_type + + boot_disk { + auto_delete = false + source = google_compute_disk.boot_disk.self_link + device_name = google_compute_disk.boot_disk.name + } + + attached_disk { + source = google_compute_disk.attached_disk.id + device_name = "attached_disk" + } + + network_interface { + network = var.network_self_link + subnetwork = var.subnetwork_self_link + } + + service_account { + email = var.service_account == null ? data.google_compute_default_service_account.default.email : var.service_account + scopes = var.scopes + } + + metadata = var.metadata + metadata_startup_script = templatefile("${path.module}/scripts/install-nfs-server.sh.tpl", { local_mounts = var.local_mounts }) + + labels = local.labels +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/outputs.tf new file mode 100644 index 0000000000..e23b94e2b2 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/outputs.tf @@ -0,0 +1,53 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +# render the content for each folder +output "network_storage" { + description = "export of all desired folder directories" + value = [for i, mount in var.local_mounts : { + remote_mount = "/exports${mount}" + local_mount = mount + fs_type = local.fs_type + mount_options = local.mount_options + server_ip = local.server_ip + client_install_runner = local.install_nfs_client_runners[i] + mount_runner = local.mount_runners[i] + } + ] +} + +output "install_nfs_client" { + description = "Script for installing NFS client" + value = file("${path.module}/scripts/install-nfs-client.sh") +} + +output "install_nfs_client_runner" { + description = "Runner to install NFS client using the startup-script module" + value = local.install_nfs_client_runners[0] +} + +output "mount_runner" { + description = <<-EOT + Runner to mount the file-system using an ansible playbook. The startup-script + module will automatically handle installation of ansible. + - id: example-startup-script + source: modules/scripts/startup-script + settings: + runners: + - $(your-fs-id.mount_runner) + ... + EOT + value = local.ansible_mount_runner +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh new file mode 100644 index 0000000000..9f842c5d7c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! "$(which mount.nfs)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || + [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + major_version=$(rpm -E "%{rhel}") + enable_repo="" + if [ "${major_version}" -eq "7" ]; then + enable_repo="base,epel" + elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then + enable_repo="baseos" + else + echo "Unsupported version of centos/RHEL/Rocky" + return 1 + fi + yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils + elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get -y install nfs-common + else + echo 'Unsuported distribution' + return 1 + fi +fi diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl new file mode 100644 index 0000000000..1b06a5f032 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl @@ -0,0 +1,35 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -ex + +if [ ! -d "/exports" ]; then # first load, format and mount the disk + # See https://cloud.google.com/compute/docs/disks/add-persistent-disk + uuid=$(uuidgen) + mkfs.ext4 -F -m 0 -U "$uuid" -E lazy_itable_init=0,lazy_journal_init=0,discard /dev/disk/by-id/google-attached_disk + + mkdir /exports + echo "UUID=$uuid /exports ext4 discard,defaults 0 0" >> /etc/fstab + mount --target /exports/ + + %{ for mount in local_mounts ~} + mkdir -p /exports${mount} + chmod 755 /exports${mount} + echo '/exports${mount} *(rw,sync,no_root_squash)' >> "/etc/exports" + %{ endfor ~} +fi + +systemctl start nfs-server rpcbind +systemctl enable nfs-server +exportfs -r diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh new file mode 100644 index 0000000000..e2509fb4a1 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SERVER_IP=$1 +REMOTE_MOUNT=$2 +LOCAL_MOUNT=$3 +FS_TYPE=$4 +MOUNT_OPTIONS=$5 + +[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" + +if [ "${FS_TYPE}" = "gcsfuse" ]; then + FS_SPEC="${REMOTE_MOUNT}" +else + FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" +fi + +SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" +EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" + +grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false +grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false +findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml new file mode 100644 index 0000000000..f7fbe58d5e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml @@ -0,0 +1,39 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Mounts the file systems specified in the metadata network_storage key + hosts: localhost + become: true + vars: + meta_key: "network_storage" + url: "http://metadata.google.internal/computeMetadata/v1/instance/attributes" + tasks: + - name: Read metadata network_storage information + ansible.builtin.uri: + url: "{{ url }}/{{ meta_key }}" + method: GET + headers: + Metadata-Flavor: "Google" + register: storage + - name: Mount file systems + ansible.posix.mount: + src: "{{ item.server_ip }}:/{{ item.remote_mount }}" + path: "{{ item.local_mount }}" + opts: "{{ item.mount_options }}" + boot: true + fstype: "{{ item.fs_type }}" + state: "mounted" + loop: "{{ storage.json }}" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/variables.tf new file mode 100644 index 0000000000..9a58da641e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/variables.tf @@ -0,0 +1,194 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "deployment_name" { + description = "Name of the HPC deployment, used as name of the NFS instance if no name is specified." + type = string +} + +variable "name" { + description = "The resource name of the instance." + type = string + default = null +} + +variable "zone" { + description = "The zone name where the NFS instance located in." + type = string +} + +variable "boot_disk_size" { + description = "Storage size in GB for the boot disk" + type = number + default = null +} + +variable "boot_disk_type" { + description = "Storage type for the boot disk" + type = string + default = null +} + +variable "create_boot_snapshot_before_destroy" { + description = "Whether to create a snapshot before destroying the boot disk" + type = bool + default = false +} + +variable "disk_size" { + description = "Storage size in GB for the NFS data disk" + type = number + default = "100" +} + +variable "type" { + description = "Storage type for the NFS data disk" + type = string + default = "pd-ssd" +} + +variable "create_snapshot_before_destroy" { + description = "Whether to create a snapshot before destroying the NFS data disk" + type = bool + default = false +} + +variable "provisioned_iops" { + description = "Provisioned IOPS for the NFS data disk if using Extreme PD or Hyperdisk Balanced/ML/Throughput" + type = number + default = null +} + +variable "provisioned_throughput" { + description = "Provisioned throughput for the NFS data disk if using Hyperdisk Balanced/Extreme" + type = number + default = null +} + +# Deprecated, replaced by instance_image +# tflint-ignore: terraform_unused_declarations +variable "image" { + description = "DEPRECATED: The VM image used by the NFS server" + type = string + default = null + + validation { + condition = var.image == null + error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." + } +} + +variable "instance_image" { + description = <<-EOD + The VM image used by the NFS server. + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + default = { + project = "cloud-hpc-image-public" + family = "hpc-rocky-linux-8" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +# Deprecated, replaced by create_snapshot_before_destroy and create_boot_snapshot_before_destroy +# tflint-ignore: terraform_unused_declarations +variable "auto_delete_disk" { + description = "DEPRECATED: Whether or not the NFS disk should be auto-deleted" + type = string + default = null + + validation { + condition = var.auto_delete_disk == null + error_message = "The 'var.auto_delete_disk' setting is broken in Cluster Toolkit versions >1.25.0 and deprecated in versions >1.48.0, please use 'var.create_snapshot_before_destroy' and 'var.create_boot_snapshot_before_destroy' instead." + } +} + +variable "network_self_link" { + description = "The self link of the network to attach the NFS VM." + type = string + default = "default" +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork to attach the NFS VM." + type = string + default = null +} + +variable "machine_type" { + description = "Type of the VM instance to use" + type = string + default = "n2d-standard-2" +} + +variable "labels" { + description = "Labels to add to the NFS instance. Key-value pairs." + type = map(string) +} + +variable "metadata" { + description = "Metadata, provided as a map" + type = map(string) + default = {} +} + +variable "service_account" { + description = "Service Account for the NFS server" + type = string + default = null +} + +variable "scopes" { + description = "Scopes to apply to the controller" + type = list(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +variable "local_mounts" { + description = "Mountpoint for this NFS compute instance" + type = list(string) + default = ["/data"] + + validation { + condition = alltrue([ + for m in var.local_mounts : substr(m, 0, 1) == "/" + ]) + error_message = "Local mountpoints have to start with '/'." + } + validation { + condition = length(var.local_mounts) > 0 + error_message = "At least one local mount must be specified in var.local_mounts." + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/versions.tf new file mode 100644 index 0000000000..8c8c39fda7 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/versions.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.14" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:nfs-server/v1.57.0" + } + + required_version = ">= 0.14.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb new file mode 100644 index 0000000000..f13726f691 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb @@ -0,0 +1,125 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "project_id = \"${project_id}\"\n", + "dataset_id = \"${dataset_id}\"\n", + "table_id = \"${table_id}\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ONI1Xo0-KtAD", + "outputId": "fb9ca475-e4ec-4cd0-e0e6-14f409eefd7a" + }, + "outputs": [], + "source": [ + "from google.cloud import bigquery\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", + "\n", + "client = bigquery.Client(project=project_id)\n", + "\n", + "df = client.query(f'''\n", + "SELECT ticker, cast(price AS FLOAT64) AS price, CAST(OFFSET as INTEGER) AS offset, start_date, end_date, iteration\n", + "FROM `{project_id}.{dataset_id}.{table_id}`,\n", + "UNNEST(simulation_results) as NUMERIC with OFFSET\n", + "WHERE epoch_time IN\n", + " # Get the latest simulation runs for each Ticker Symbol\n", + "(SELECT MAX(epoch_time) FROM `{project_id}.{dataset_id}.{table_id}` GROUP BY ticker)\n", + "'''\n", + ").to_dataframe()\n", + "# Display the data\n", + "df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Define a function to plot the data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "def plot_ticker(t,df):\n", + "\n", + " dtf = df[(df.ticker==t) &(df.offset == 250)].price.describe(include=[np.float64], percentiles=[.05, .01, .001])\n", + " cellText = []\n", + " for v in dtf.values:\n", + " cellText.append([v])\n", + " \n", + " pltf = df[df.ticker==t].pivot(index='offset', columns='iteration', values='price')\n", + " \n", + " fig = plt.figure(figsize=(10,5))\n", + " ax1 = fig.add_subplot(122)\n", + " pltf.plot(legend=False, ax=ax1, xlabel='Time(days)', ylabel='US$', title=f\"{ df[(df.ticker == t) & (df.offset == 0) & (df.iteration == 4)]}\")\n", + " ax2 = fig.add_subplot(121)\n", + " font_size=10\n", + " bbox=[0, 0, .5, 1]\n", + " ax2.axis('off')\n", + " mpl_table = ax2.table(cellText = cellText, rowLabels=dtf.index.values, bbox=bbox)\n", + " mpl_table.auto_set_font_size(False)\n", + " mpl_table.set_fontsize(font_size)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 808 + }, + "id": "jvBmb_KceX7z", + "outputId": "42a3ba9f-b68f-4c7b-d928-0fedeed9216c" + }, + "outputs": [], + "source": [ + "ticker_list = df.ticker.unique()\n", + "for t in ticker_list:\n", + " plot_ticker(t,df)" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md new file mode 100644 index 0000000000..e54893a1bb --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md @@ -0,0 +1,97 @@ +## Description + +Copy files to a target GCS bucket. + +Primarily used for FSI - MonteCarlo Tutorial **[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: +../docs/tutorials/fsi-montecarlo-on-batch/README.md + +## Usage +This copies the module files to the specified GCS bucket. It is expected that +the bucket will be mounted on the target VM. + +Some of the files are templates, and `main.tf` translates the files with the +passed variable values. This way the user does not have to change things like +pointing to the correct bigquery table or adding in the project_id. + +```yaml + - id: fsi_tutorial_files + source: community/modules/files/fsi-montecarlo-on-batch + use: [bq-dataset, bq-table, fsi_bucket, pubsub_topic] +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 3.83 | +| [http](#requirement\_http) | ~> 3.0 | +| [random](#requirement\_random) | ~> 3.0 | +| [template](#requirement\_template) | ~> 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [http](#provider\_http) | ~> 3.0 | +| [random](#provider\_random) | ~> 3.0 | +| [template](#provider\_template) | ~> 2.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.get_iteration_sh](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.get_mc_reqs](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.get_requirements](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.ipynb_obj_fsi](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.mc_obj_yaml](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.mc_run](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.run_batch_py](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [http_http.batch_py](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | +| [http_http.batch_requirements](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | +| [template_file.ipynb_fsi](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | +| [template_file.mc_run_py](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | +| [template_file.mc_run_yaml](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [dataset\_id](#input\_dataset\_id) | Bigquery dataset id | `string` | n/a | yes | +| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | Bucket name | `string` | `null` | no | +| [project\_id](#input\_project\_id) | ID of project in which GCS bucket will be created. | `string` | n/a | yes | +| [region](#input\_region) | Region to run project | `string` | n/a | yes | +| [table\_id](#input\_table\_id) | Bigquery table id | `string` | n/a | yes | +| [topic\_id](#input\_topic\_id) | Pubsub Topic Name | `string` | n/a | yes | +| [topic\_schema](#input\_topic\_schema) | Pubsub Topic schema | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh new file mode 100644 index 0000000000..50aa865a31 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ticker=("GOOG" "AMZN" "MSFT" "NVDA" "META" "TSLA" "PEP" "COST") +echo "BI: $BATCH_TASK_INDEX" +echo "TI: ${ticker[$BATCH_TASK_INDEX]}" +python3 -m pip install -r /mnt/disks/fsi/mc_run_reqs.txt +python3 /mnt/disks/fsi/mc_run.py \ + --ticker "${ticker[$BATCH_TASK_INDEX]}" \ + --iterations 500 \ + --start_date 2022-01-01 diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf new file mode 100644 index 0000000000..83dc7fe9cf --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf @@ -0,0 +1,102 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + bucket = replace(var.gcs_bucket_path, "gs://", "") +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +data "template_file" "mc_run_py" { + template = file("${path.module}/mc_run.tpl.py") + vars = { + project_id = var.project_id + topic_id = var.topic_id + topic_schema = var.topic_schema + dataset_id = var.dataset_id + table_id = var.table_id + } +} + +resource "google_storage_bucket_object" "mc_run" { + name = "mc_run.py" + content = data.template_file.mc_run_py.rendered + bucket = local.bucket +} + +data "template_file" "mc_run_yaml" { + template = file("${path.module}/mc_run.tpl.yaml") + vars = { + project_id = var.project_id + bucket_name = local.bucket + region = var.region + } +} + +resource "google_storage_bucket_object" "mc_obj_yaml" { + name = "mc_run.yaml" + content = data.template_file.mc_run_yaml.rendered + bucket = local.bucket +} + +data "template_file" "ipynb_fsi" { + template = file("${path.module}/FSI_MonteCarlo.ipynb") + vars = { + project_id = var.project_id + dataset_id = var.dataset_id + table_id = var.table_id + } +} +resource "google_storage_bucket_object" "ipynb_obj_fsi" { + name = "FSI_MonteCarlo.ipynb" + content = data.template_file.ipynb_fsi.rendered + bucket = local.bucket +} + +data "http" "batch_py" { + url = "https://raw.githubusercontent.com/GoogleCloudPlatform/scientific-computing-examples/main/python-batch/batch.py" +} + +resource "google_storage_bucket_object" "run_batch_py" { + name = "batch.py" + content = data.http.batch_py.response_body + bucket = local.bucket +} + +data "http" "batch_requirements" { + url = "https://raw.githubusercontent.com/GoogleCloudPlatform/scientific-computing-examples/main/python-batch/requirements.txt" +} + +resource "google_storage_bucket_object" "get_requirements" { + name = "requirements.txt" + content = data.http.batch_requirements.response_body + bucket = local.bucket +} + +resource "google_storage_bucket_object" "get_iteration_sh" { + name = "iteration.sh" + content = file("${path.module}/iteration.sh") + bucket = local.bucket +} + +resource "google_storage_bucket_object" "get_mc_reqs" { + name = "mc_run_reqs.txt" + content = file("${path.module}/mc_run_reqs.txt") + bucket = local.bucket +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py new file mode 100644 index 0000000000..4e0a64e363 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Run MC simulation for VaR portfolio risk +""" + +import avro.schema +import io +import google.auth +import numpy +import time +import yfinance as yf + +from absl import app +from absl import flags +from avro.io import DatumWriter, BinaryEncoder, BinaryDecoder, DatumReader +from datetime import datetime +from datetime import timedelta +from google.cloud import pubsub_v1, bigquery +from google.cloud.pubsub import SchemaServiceClient + +PROJECT_ID = '${project_id}' +INCOMING_TOPIC_ID = '${topic_id}' +INCOMING_TOPIC_SCHEMA = '${topic_schema}' +DATASET_ID = '${dataset_id}' +TABLE_ID = '${table_id}' + + +FLAGS = flags.FLAGS + +flags.DEFINE_string("ticker", 'GOOG', "Nasdaq Stock Ticker to run, default GOOG") +flags.DEFINE_string("start_date", '2022-01-01' , "Start data for data query, default 2022-01-01") +flags.DEFINE_integer("calendar_days", 365 , "How many calendar days to include in the calculation") +flags.DEFINE_integer("epoch_time", f'{int(time.time())}' , "Epoch time, number of seconds since January 1st, 1970 at 00:00:00 UTC.") +flags.DEFINE_integer("iterations", 100 , "Number of iterations to run.") +flags.DEFINE_boolean("print_raw", False, "Dump raw data.") + +class VaRSimulator: + + def __init__(self): + pass + + def get_data(self): + self.get_historical_data_yahoo() + + def get_historical_data_yahoo(self): + + # get historical market data: https://pypi.org/project/yfinance/ + + self.raw_data = yf.Ticker(self.ticker).history(start=self.start_date, end=self.end_date ) + self.data = self.raw_data.Close + + def print_raw(self): + print(self.get_stats()) + print(type(self.raw_data)) + print(self.raw_data) + + def get_stats(self): + close = self.data + self.first = close[0] + self.last = close[-1] + self.trading_days = len(close) + self.cagr = (self.last / self.first) ** (365.0/self.calendar_days) -1.0 + self.volatility = self.data.pct_change().std() + return(self.first, self.last, self.trading_days, self.cagr, self.volatility) + + def run_simulation(self): + + returns = numpy.random.normal(self.cagr/self.trading_days, self.volatility, self.trading_days) + 1 + returns = numpy.insert(returns,0,1.0) + self.simulation_results = self.last * returns.cumprod() + return(self.simulation_results) + + def create_object(self): + self.object = { + "ticker": self.ticker, + "epoch_time": self.epoch_time, + "iteration": self.iteration, + "start_date": self.start_date, + "end_date": self.end_date, + "simulation_results": list(map(lambda x: {"price":x}, self.simulation_results)) + } + return(self.object) + + +class PubsubToBiquery: + + def __init__(self): + + the_time = int(time.time()) + + self.project_id = PROJECT_ID + + self.publisher_client = pubsub_v1.PublisherClient() + self.topic_path = self.publisher_client.topic_path(self.project_id, INCOMING_TOPIC_ID) + + self.schema_client = SchemaServiceClient() + self.schema_path = self.schema_client.schema_path(self.project_id, INCOMING_TOPIC_SCHEMA) + + pubsub_schema = self.schema_client.get_schema(request={"name": self.schema_path}) + avro_schema = avro.schema.parse(pubsub_schema.definition) + + self.writer = DatumWriter(avro_schema) + + + def publish_record(self,record): + + byte_stream = io.BytesIO() + encoder = BinaryEncoder(byte_stream) + self.writer.write(record, encoder) + data = byte_stream.getvalue() + byte_stream.flush() + future = self.publisher_client.publish(self.topic_path, data) + if(FLAGS.print_raw): + print(f"Published message ID: {future.result()}") + + +def main(argv): + + vr = VaRSimulator() + pbbq = PubsubToBiquery() + + vr.ticker =FLAGS.ticker + vr.start_date =FLAGS.start_date + vr.end_date =f'{(datetime.strptime(FLAGS.start_date,"%Y-%m-%d") + timedelta(days = FLAGS.calendar_days)).date()}' + vr.calendar_days = FLAGS.calendar_days + vr.epoch_time = FLAGS.epoch_time + vr.iteration = 1 + + vr.get_data() + vr.get_stats() + + for i in range(FLAGS.iterations): + vr.iteration = i + vr.run_simulation() + pbbq.publish_record(vr.create_object()) + + if(FLAGS.print_raw): + vr.print_raw() + + +if __name__ == "__main__": + """ This is executed when run from the command line """ + app.run(main) diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml new file mode 100644 index 0000000000..7f7de4840b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml @@ -0,0 +1,36 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +project_id: "${project_id}" +region: "${region}" + +job_prefix: 'fsi-' +machine_type: "n2-standard-2" +volumes: +- {bucket_name: "${bucket_name}", gcs_path: "/mnt/disks/fsi"} + +container: + image_uri: "python" + entry_point: "/bin/bash" + commands: ["/mnt/disks/fsi/iteration.sh", "$BATCH_TASK_INDEX"] + +task_count: 8 #optional +parallelism: 4 #optional +task_count_per_node: 2 #optional +cpu_milli: 1000 #optional +memory_mib: 102400 #optional + + +labels: + env: "monte" + type: "carlo" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt new file mode 100644 index 0000000000..105ed70ad2 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt @@ -0,0 +1,9 @@ +absl-py +avro +google-auth +google-cloud +google-cloud-batch +google-cloud-pubsub +google-cloud-bigquery +yfinance +PyYAML diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml new file mode 100644 index 0000000000..268c8faa9a --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [storage.googleapis.com] diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf new file mode 100644 index 0000000000..eddf3c9478 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which GCS bucket will be created." + type = string +} + +variable "gcs_bucket_path" { + description = "Bucket name" + type = string + default = null +} + +variable "topic_id" { + description = "Pubsub Topic Name" + type = string +} + +variable "topic_schema" { + description = "Pubsub Topic schema" + type = string +} + +variable "dataset_id" { + description = "Bigquery dataset id" + type = string +} + +variable "table_id" { + description = "Bigquery table id" + type = string +} + +variable "region" { + description = "Region to run project" + type = string +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf new file mode 100644 index 0000000000..07270f9cfe --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf @@ -0,0 +1,43 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + http = { + source = "hashicorp/http" + version = "~> 3.0" + } + template = { + source = "hashicorp/template" + version = "~> 2.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.57.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.57.0" + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md new file mode 100644 index 0000000000..ae8462d763 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md @@ -0,0 +1,100 @@ +# Module: Slurm Instance + + + +- [Module: Slurm Instance](#module-slurm-instance) + - [Overview](#overview) + - [Module API](#module-api) + + + +## Overview + +This module creates a [compute instance](../../../../docs/glossary.md#vm) from +[instance template](../../../../docs/glossary.md#instance-template) for a +[Slurm cluster](../slurm_cluster/README.md). + +> **NOTE:** This module is only intended to be used by Slurm modules. For +> general usage, please consider using: +> +> - [terraform-google-modules/vm/google//modules/compute_instance](https://registry.terraform.io/modules/terraform-google-modules/vm/google/latest/submodules/compute_instance). +> **WARNING:** The source image is not modified. Make sure to use a compatible +> source image. + +## Module API + +For the terraform module API reference, please see +[README_TF.md](./README_TF.md). + + +Copyright (C) SchedMD LLC. +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~> 1.0 | +| [google](#requirement\_google) | >= 3.43 | +| [null](#requirement\_null) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.43 | +| [null](#provider\_null) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_instance_from_template.slurm_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_from_template) | resource | +| [null_resource.replace_trigger](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_instance_template.base](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance_template) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    network            = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    queue_count        = optional(number)
    stack_type         = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
  }))
| `[]` | no | +| [hostname](#input\_hostname) | Hostname of instances | `string` | n/a | yes | +| [instance\_template](#input\_instance\_template) | Instance template self\_link used to create compute instances | `string` | n/a | yes | +| [network](#input\_network) | Network to deploy to. Only one of network or subnetwork should be specified. | `string` | `""` | no | +| [num\_instances](#input\_num\_instances) | Number of instances to create. This value is ignored if static\_ips is provided. | `number` | `1` | no | +| [project\_id](#input\_project\_id) | The GCP project ID | `string` | `null` | no | +| [region](#input\_region) | Region where the instances should be created. | `string` | `null` | no | +| [replace\_trigger](#input\_replace\_trigger) | Trigger value to replace the instances. | `string` | `""` | no | +| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances | `list(string)` | `[]` | no | +| [subnetwork](#input\_subnetwork) | Subnet to deploy to. Only one of network or subnetwork should be specified. | `string` | `""` | no | +| [subnetwork\_project](#input\_subnetwork\_project) | The project that subnetwork belongs to | `string` | `null` | no | +| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be spread across available zones in the region. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [available\_zones](#output\_available\_zones) | List of available zones in region | +| [instances\_details](#output\_instances\_details) | List of all details for compute instances | +| [instances\_self\_links](#output\_instances\_self\_links) | List of self-links for compute instances | +| [names](#output\_names) | List of available zones in region | +| [slurm\_instances](#output\_slurm\_instances) | List of all resource objects for compute instances | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf new file mode 100644 index 0000000000..2af9008a0e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf @@ -0,0 +1,126 @@ +/** + * Copyright (C) SchedMD LLC. + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +########## +# LOCALS # +########## + +locals { + num_instances = length(var.static_ips) == 0 ? var.num_instances : length(var.static_ips) + + # local.static_ips is the same as var.static_ips with a dummy element appended + # at the end of the list to work around "list does not have any elements so cannot + # determine type" error when var.static_ips is empty + static_ips = concat(var.static_ips, ["NOT_AN_IP"]) + + network_interfaces = [for index in range(local.num_instances) : + concat([ + { + access_config = var.access_config + alias_ip_range = [] + ipv6_access_config = [] + network = var.network + network_ip = length(var.static_ips) == 0 ? "" : element(local.static_ips, index) + nic_type = null + queue_count = null + stack_type = null + subnetwork = var.subnetwork + subnetwork_project = var.subnetwork_project + } + ], + var.additional_networks + ) + ] +} + +################ +# DATA SOURCES # +################ + +data "google_compute_zones" "available" { + project = var.project_id + region = var.region +} + +data "google_compute_instance_template" "base" { + project = var.project_id + name = var.instance_template +} + +############# +# INSTANCES # +############# +resource "null_resource" "replace_trigger" { + triggers = { + trigger = var.replace_trigger + } +} + +# TODO: `internal/slurm-gcp/login` is ONLY user of `internal/slurm-gcp/instance` +# Remove this module, add functionality (+ prune generality) to the login module directly. +resource "google_compute_instance_from_template" "slurm_instance" { + count = local.num_instances + name = format("%s-%s", var.hostname, format("%03d", count.index + 1)) + project = var.project_id + zone = var.zone == null ? data.google_compute_zones.available.names[count.index % length(data.google_compute_zones.available.names)] : var.zone + + allow_stopping_for_update = true + + dynamic "network_interface" { + for_each = local.network_interfaces[count.index] + iterator = nic + content { + dynamic "access_config" { + for_each = nic.value.access_config + content { + nat_ip = access_config.value.nat_ip + network_tier = access_config.value.network_tier + } + } + dynamic "alias_ip_range" { + for_each = nic.value.alias_ip_range + content { + ip_cidr_range = alias_ip_range.value.ip_cidr_range + subnetwork_range_name = alias_ip_range.value.subnetwork_range_name + } + } + dynamic "ipv6_access_config" { + for_each = nic.value.ipv6_access_config + iterator = access_config + content { + network_tier = access_config.value.network_tier + } + } + network = nic.value.network + network_ip = nic.value.network_ip + nic_type = nic.value.nic_type + queue_count = nic.value.queue_count + subnetwork = nic.value.subnetwork + subnetwork_project = nic.value.subnetwork_project + } + } + + source_instance_template = data.google_compute_instance_template.base.self_link + # Due to https://github.com/hashicorp/terraform-provider-google/issues/21693 + # we have to explicitly override instance labels instead of inheriting them from template. + labels = data.google_compute_instance_template.base.labels + + + lifecycle { + replace_triggered_by = [null_resource.replace_trigger.id] + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf new file mode 100644 index 0000000000..4eba78a7e8 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf @@ -0,0 +1,41 @@ +/** + * Copyright (C) SchedMD LLC. + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "slurm_instances" { + description = "List of all resource objects for compute instances" + value = google_compute_instance_from_template.slurm_instance +} + +output "instances_self_links" { + description = "List of self-links for compute instances" + value = google_compute_instance_from_template.slurm_instance[*].self_link +} + +output "instances_details" { + description = "List of all details for compute instances" + value = google_compute_instance_from_template.slurm_instance[*] +} + +output "available_zones" { + description = "List of available zones in region" + value = data.google_compute_zones.available.names +} + +output "names" { + description = "List of available zones in region" + value = google_compute_instance_from_template.slurm_instance[*].name +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf new file mode 100644 index 0000000000..11111a2c05 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf @@ -0,0 +1,119 @@ +/** + * Copyright (C) SchedMD LLC. + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "The GCP project ID" + default = null +} + +variable "network" { + description = "Network to deploy to. Only one of network or subnetwork should be specified." + type = string + default = "" +} + +variable "subnetwork" { + description = "Subnet to deploy to. Only one of network or subnetwork should be specified." + type = string + default = "" +} + +variable "subnetwork_project" { + description = "The project that subnetwork belongs to" + type = string + default = null +} + +variable "hostname" { + description = "Hostname of instances" + type = string +} + +variable "additional_networks" { + description = "Additional network interface details for GCE, if any." + default = [] + type = list(object({ + access_config = optional(list(object({ + nat_ip = string + network_tier = string + })), []) + alias_ip_range = optional(list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })), []) + ipv6_access_config = optional(list(object({ + network_tier = string + })), []) + network = optional(string) + network_ip = optional(string, "") + nic_type = optional(string) + queue_count = optional(number) + stack_type = optional(string) + subnetwork = optional(string) + subnetwork_project = optional(string) + })) + nullable = false +} + +variable "static_ips" { + description = "List of static IPs for VM instances" + type = list(string) + default = [] +} + +variable "access_config" { + description = "Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet." + type = list(object({ + nat_ip = string + network_tier = string + })) + default = [] +} + +variable "num_instances" { + description = "Number of instances to create. This value is ignored if static_ips is provided." + type = number + default = 1 +} + +variable "instance_template" { + description = "Instance template self_link used to create compute instances" + type = string +} + +variable "region" { + description = "Region where the instances should be created." + type = string + default = null +} + +variable "zone" { + description = "Zone where the instances should be created. If not specified, instances will be spread across available zones in the region." + type = string + default = null +} + +######### +# SLURM # +######### + +variable "replace_trigger" { + description = "Trigger value to replace the instances." + type = string + default = "" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf new file mode 100644 index 0000000000..a3e84c09bf --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf @@ -0,0 +1,31 @@ +/** + * Copyright (C) SchedMD LLC. + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = "~> 1.0" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.43" + } + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md new file mode 100644 index 0000000000..8b57f96642 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md @@ -0,0 +1,86 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~> 1.0 | +| [local](#requirement\_local) | ~> 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [local](#provider\_local) | ~> 2.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [instance\_template](#module\_instance\_template) | ../internal_instance_template | n/a | + +## Resources + +| Name | Type | +|------|------| +| [local_file.startup](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | +| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    source                     = optional(string)
    disk_name                  = optional(string)
    device_name                = string
    disk_type                  = optional(string)
    disk_size_gb               = optional(number)
    disk_labels                = map(string)
    auto_delete                = bool
    boot                       = bool
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| n/a | yes | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `virtio_enabled` setting will only enable VirtioNet and will not enable TIER\_1.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image that supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | +| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | +| [disk\_labels](#input\_disk\_labels) | Labels to be assigned to boot disk, provided as a map. | `map(string)` | `{}` | no | +| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `100` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either pd-ssd, local-ssd, or pd-standard. | `string` | `"pd-standard"` | no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [gpu](#input\_gpu) | GPU information. Type and count of GPU to attach to the instance template. See
https://cloud.google.com/compute/docs/gpus more details.
- type : the GPU type
- count : number of GPUs | 
object({
    type  = string
    count = number
  })
| `null` | no | +| [internal\_startup\_script](#input\_internal\_startup\_script) | FOR INTERNAL TOOLKIT USAGE ONLY. | `string` | `null` | no | +| [labels](#input\_labels) | Labels, provided as a map | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"n1-standard-1"` | no | +| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | +| [name\_prefix](#input\_name\_prefix) | Prefix for template resource. | `string` | `"default"` | no | +| [network](#input\_network) | The name or self\_link of the network to attach this interface to. Use network
attribute for Legacy or Auto subnetted networks and subnetwork for custom
subnetted networks. | `string` | `null` | no | +| [network\_ip](#input\_network\_ip) | Private IP address to assign to the instance if desired. | `string` | `""` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy | `string` | `"MIGRATE"` | no | +| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [provisioning\_model](#input\_provisioning\_model) | The provisioning model of the instance | `string` | `null` | no | +| [region](#input\_region) | Region where the instance template should be created. | `string` | n/a | yes | +| [reservation\_affinity](#input\_reservation\_affinity) | Specifies the reservations that this instance can consume from. | `object({ type = string })` | `null` | no | +| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [service\_account](#input\_service\_account) | Service account to attach to the instances. See
'main.tf:local.service\_account' for the default. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used for resource naming. | `string` | n/a | yes | +| [slurm\_instance\_role](#input\_slurm\_instance\_role) | Slurm instance type. Must be one of: controller; login; compute; or null. | `string` | n/a | yes | +| [source\_image](#input\_source\_image) | Source disk image. | `string` | `""` | no | +| [source\_image\_family](#input\_source\_image\_family) | Source image family. | `string` | `""` | no | +| [source\_image\_project](#input\_source\_image\_project) | Project where the source image comes from. If it is not provided, the provider project is used. | `string` | `""` | no | +| [spot](#input\_spot) | Provision as a SPOT preemptible instance.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `bool` | `false` | no | +| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach this interface to. The subnetwork must
exist in the same region this instance will be created in. Either network or
subnetwork must be provided. | `string` | `null` | no | +| [subnetwork\_project](#input\_subnetwork\_project) | The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used. | `string` | `null` | no | +| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | +| [termination\_action](#input\_termination\_action) | Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instance\_template](#output\_instance\_template) | Instance template details | +| [labels](#output\_labels) | Labels attached to the instance template | +| [name](#output\_name) | Name of instance template | +| [self\_link](#output\_self\_link) | Self\_link of instance template | +| [service\_account](#output\_service\_account) | Service account object, includes email and scopes. | +| [tags](#output\_tags) | Tags that will be associated with instance(s) | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted new file mode 100644 index 0000000000..fb6191eec6 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted @@ -0,0 +1,161 @@ +#!/bin/bash +# Copyright (C) SchedMD LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +SLURM_DIR=/slurm +FLAGFILE=$SLURM_DIR/slurm_configured_do_not_remove +SCRIPTS_DIR=$SLURM_DIR/scripts +if [[ -z "$HOME" ]]; then + # google-startup-scripts.service lacks environment variables + HOME="$(getent passwd "$(whoami)" | cut -d: -f6)" +fi + +# Temporary workaround for transition period when some of older images +# don't have "baked in" python yet. +# TODO: Remove +SLURM_PY="/slurm/python/venv/bin/python3.13" +SYSTEM_PY="/usr/bin/python3" +if [[ ! -e "$SLURM_PY" ]]; then + echo "Symlink $SLURM_PY does not exist. Creating symlink to $SYSTEM_PY" + mkdir -p /slurm/python/venv/bin + ln -s "$SYSTEM_PY" "$SLURM_PY" +fi + +METADATA_SERVER="metadata.google.internal" +URL="http://$METADATA_SERVER/computeMetadata/v1" +CURL="curl -sS --fail --header Metadata-Flavor:Google" + +PING_METADATA="ping -q -w1 -c1 $METADATA_SERVER" +echo "INFO: $PING_METADATA" +for i in $(seq 10); do + [ $i -gt 1 ] && sleep 5; + $PING_METADATA > /dev/null && s=0 && break || s=$?; + echo "ERROR: Failed to contact metadata server, will retry" +done +if [ $s -ne 0 ]; then + echo "ERROR: Unable to contact metadata server, aborting" + wall -n '*** Slurm setup failed in the startup script! see `journalctl -u google-startup-scripts` ***' + exit 1 +else + echo "INFO: Successfully contacted metadata server" +fi + +PING_GOOGLE="ping -q -w1 -c1 8.8.8.8" +echo "INFO: $PING_GOOGLE" +for i in $(seq 5); do + [ $i -gt 1 ] && sleep 2; + $PING_GOOGLE > /dev/null && s=0 && break || s=$?; + echo "failed to ping Google DNS, will retry" +done +if [ $s -ne 0 ]; then + echo "WARNING: No internet access detected" +else + echo "INFO: Internet access detected" +fi + +mkdir -p $SCRIPTS_DIR +UNIVERSE_DOMAIN="$($CURL $URL/instance/attributes/universe_domain)" +BUCKET="$($CURL $URL/instance/attributes/slurm_bucket_path)" +if [[ -z $BUCKET ]]; then + echo "ERROR: No bucket path detected." + exit 1 +fi + +SCRIPTS_ZIP="$HOME/slurm-gcp-scripts.zip" +export CLOUDSDK_CORE_UNIVERSE_DOMAIN="$UNIVERSE_DOMAIN" +until gcloud storage cp "$BUCKET/slurm-gcp-devel.zip" "$SCRIPTS_ZIP"; do + echo "WARN: Could not download SlurmGCP scripts, retrying in 5 seconds." + # Remove marker used to determine if gcloud is being used in a GCE VM. + # This can get mistakenly set to False in some cases. + rm -f /root/.config/gcloud/gce + sleep 5 +done +unzip -o "$SCRIPTS_ZIP" -d "$SCRIPTS_DIR" +rm -rf "$SCRIPTS_ZIP" + +#temporary hack to not make the script fail on TPU vm +chown slurm:slurm -R "$SCRIPTS_DIR" || true +chmod 700 -R "$SCRIPTS_DIR" + + +if [ -f $FLAGFILE ]; then + echo "WARNING: Slurm was previously configured, quitting" + exit 0 +fi +touch $FLAGFILE + +function tpu_setup { + #allow the following command to fail, as this attribute does not exist for regular nodes + docker_image=$($CURL $URL/instance/attributes/slurm_docker_image 2> /dev/null || true) + if [ -z $docker_image ]; then #Not a tpu node, do not do anything + return + fi + if [ "$OS_ENV" == "slurm_container" ]; then #Already inside the slurm container, we should continue starting + return + fi + + #given a input_string like "WORKER_0:Joseph;WORKER_1:richard;WORKER_2:edward;WORKER_3:john" and a number 1, this function will print richard + parse_metadata() { + local number=$1 + local input_string=$2 + local word=$(echo "$input_string" | awk -v n="$number" -F ':|;' '{ for (i = 1; i <= NF; i+=2) if ($(i) == "WORKER_"n) print $(i+1) }') + echo "$word" + } + + input_string=$($CURL $URL/instance/attributes/slurm_names) + worker_id=$($CURL $URL/instance/attributes/tpu-env | awk '/WORKER_ID/ {print $2}' | tr -d \') + real_name=$(parse_metadata $worker_id $input_string) + + #Prepare to docker pull with gcloud + mkdir -p /root/.docker + cat << EOF > /root/.docker/config.json +{ + "credHelpers": { + "gcr.io": "gcloud", + "us-docker.pkg.dev": "gcloud" + } +} +EOF + #cgroup detection + CGV=1 + CGROUP_FLAGS="-v /sys/fs/cgroup:/sys/fs/cgroup:rw" + if [ -f /sys/fs/cgroup/cgroup.controllers ]; then #CGV2 + CGV=2 + fi + if [ $CGV == 2 ]; then + CGROUP_FLAGS="--cgroup-parent=docker.slice --cgroupns=private --tmpfs /run --tmpfs /run/lock --tmpfs /tmp" + if [ ! -f /etc/systemd/system/docker.slice ]; then #In case that there is no slice prepared for hosting the containers create it + printf "[Unit]\nDescription=docker slice\nBefore=slices.target\n[Slice]\nCPUAccounting=true\nMemoryAccounting=true" > /etc/systemd/system/docker.slice + systemctl start docker.slice + fi + fi + #for the moment always use --privileged, as systemd might not work properly otherwise + TPU_FLAGS="--privileged" + # TPU_FLAGS="--cap-add SYS_RESOURCE --device /dev/accel0 --device /dev/accel1 --device /dev/accel2 --device /dev/accel3" + # if [ $CGV == 2 ]; then #In case that we are in CGV2 for systemd to work correctly for the moment we go with privileged + # TPU_FLAGS="--privileged" + # fi + + docker run -d $CGROUP_FLAGS $TPU_FLAGS --net=host --name=slurmd --hostname=$real_name --entrypoint=/usr/bin/systemd --restart unless-stopped $docker_image + exit 0 +} + +tpu_setup #will do nothing for normal nodes or the container spawned inside TPU + +echo "INFO: Running python cluster setup script" +SETUP_SCRIPT_FILE=$SCRIPTS_DIR/setup.py +chmod +x $SETUP_SCRIPT_FILE +exec $SETUP_SCRIPT_FILE diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf new file mode 100644 index 0000000000..3ec9f466de --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf @@ -0,0 +1,164 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +########## +# LOCALS # +########## + +locals { + additional_disks = [ + for disk in var.additional_disks : { + disk_name = disk.disk_name + device_name = disk.device_name + auto_delete = disk.auto_delete + source = disk.source + boot = disk.boot + disk_size_gb = disk.disk_size_gb + disk_type = disk.disk_type + disk_labels = merge( + disk.disk_labels, + { + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = var.slurm_instance_role + }, + ) + disk_resource_manager_tags = disk.disk_resource_manager_tags + } + ] + + service_account = { + email = try(var.service_account.email, null) + scopes = try(var.service_account.scopes, ["https://www.googleapis.com/auth/cloud-platform"]) + } + + source_image_family = ( + var.source_image_family != "" && var.source_image_family != null + ? var.source_image_family + : "slurm-gcp-6-9-hpc-rocky-linux-8" + ) + source_image_project = ( + var.source_image_project != "" && var.source_image_project != null + ? var.source_image_project + : "projects/schedmd-slurm-public/global/images/family" + ) + + source_image = ( + var.source_image != null + ? var.source_image + : "" + ) + + + name_prefix = "${var.slurm_cluster_name}-${var.slurm_instance_role}-${var.name_prefix}" + + total_egress_bandwidth_tier = var.bandwidth_tier == "tier_1_enabled" ? "TIER_1" : "DEFAULT" + + nic_type_map = { + platform_default = null + virtio_enabled = "VIRTIO_NET" + gvnic_enabled = "GVNIC" + tier_1_enabled = "GVNIC" + } + nic_type = lookup(local.nic_type_map, var.bandwidth_tier, null) + + labels = merge(var.labels, + { + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = var.slurm_instance_role + }, + ) +} + +######## +# DATA # +######## + +data "local_file" "startup" { + filename = "${path.module}/files/startup_sh_unlinted" +} + +############ +# TEMPLATE # +############ + +module "instance_template" { + source = "../internal_instance_template" + + project_id = var.project_id + + # Network + can_ip_forward = var.can_ip_forward + network_ip = var.network_ip + network = var.network + nic_type = local.nic_type + region = var.region + subnetwork_project = var.subnetwork_project + subnetwork = var.subnetwork + tags = var.tags + total_egress_bandwidth_tier = local.total_egress_bandwidth_tier + additional_networks = var.additional_networks + access_config = var.access_config + + # Instance + machine_type = var.machine_type + min_cpu_platform = var.min_cpu_platform + name_prefix = local.name_prefix + gpu = var.gpu + service_account = local.service_account + shielded_instance_config = var.shielded_instance_config + advanced_machine_features = var.advanced_machine_features + enable_confidential_vm = var.enable_confidential_vm + enable_shielded_vm = var.enable_shielded_vm + preemptible = var.preemptible + spot = var.spot + on_host_maintenance = var.on_host_maintenance + labels = local.labels + instance_termination_action = var.termination_action + resource_manager_tags = var.resource_manager_tags + + # Metadata + startup_script = coalesce(var.internal_startup_script, data.local_file.startup.content) + metadata = merge( + var.metadata, + { + enable-oslogin = upper(var.enable_oslogin) + slurm_bucket_path = var.slurm_bucket_path + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = var.slurm_instance_role + }, + ) + + # Image + source_image_project = local.source_image_project + source_image_family = local.source_image_family + source_image = local.source_image + + # Disk + disk_type = var.disk_type + disk_size_gb = var.disk_size_gb + auto_delete = var.disk_auto_delete + disk_labels = merge( + { + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = var.slurm_instance_role + }, + var.disk_labels, + ) + disk_resource_manager_tags = var.disk_resource_manager_tags + additional_disks = local.additional_disks + + max_run_duration = var.max_run_duration + provisioning_model = var.provisioning_model + reservation_affinity = var.reservation_affinity +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf new file mode 100644 index 0000000000..65da41052e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf @@ -0,0 +1,43 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "instance_template" { + description = "Instance template details" + value = module.instance_template +} + +output "self_link" { + description = "Self_link of instance template" + value = module.instance_template.self_link +} + +output "name" { + description = "Name of instance template" + value = module.instance_template.name +} + +output "tags" { + description = "Tags that will be associated with instance(s)" + value = module.instance_template.tags +} + +output "service_account" { + description = "Service account object, includes email and scopes." + value = module.instance_template.service_account +} + +output "labels" { + description = "Labels attached to the instance template" + value = local.labels +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf new file mode 100644 index 0000000000..35dd9c376f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf @@ -0,0 +1,431 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +########### +# GENERAL # +########### + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "on_host_maintenance" { + type = string + description = "Instance availability Policy" + default = "MIGRATE" +} + +variable "labels" { + type = map(string) + description = "Labels, provided as a map" + default = {} +} + +variable "enable_oslogin" { + type = bool + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >=0.13.0 | +| [google](#requirement\_google) | >= 3.88 | +| [google-beta](#requirement\_google-beta) | >= 6.13.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.88 | +| [google-beta](#provider\_google-beta) | >= 6.13.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_instance_template.tpl](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance_template) | resource | +| [google_project.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | +| [additional\_disks](#input\_additional\_disks) | List of maps of additional disks. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#disk_name | 
list(object({
    source                     = optional(string)
    disk_name                  = optional(string)
    device_name                = string
    auto_delete                = bool
    boot                       = bool
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = map(string)
    disk_resource_manager_tags = map(string)
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| n/a | yes | +| [alias\_ip\_range](#input\_alias\_ip\_range) | An array of alias IP ranges for this network interface. Can only be specified for network interfaces on subnet-mode networks.
ip\_cidr\_range: The IP CIDR range represented by this alias IP range. This IP CIDR range must belong to the specified subnetwork and cannot contain IP addresses reserved by system or used by other network interfaces. At the time of writing only a netmask (e.g. /24) may be supplied, with a CIDR format resulting in an API error.
subnetwork\_range\_name: The subnetwork secondary range name specifying the secondary range from which to allocate the IP CIDR range for this alias IP range. If left unspecified, the primary range of the subnetwork will be used. | 
object({
    ip_cidr_range         = string
    subnetwork_range_name = string
  })
| `null` | no | +| [auto\_delete](#input\_auto\_delete) | Whether or not the boot disk should be auto-deleted | `string` | `"true"` | no | +| [automatic\_restart](#input\_automatic\_restart) | (Optional) Specifies whether the instance should be automatically restarted if it is terminated by Compute Engine (not terminated by a user). | `bool` | `true` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example | `string` | `"false"` | no | +| [disk\_encryption\_key](#input\_disk\_encryption\_key) | The id of the encryption key that is stored in Google Cloud KMS to use to encrypt all the disks on this instance | `string` | `null` | no | +| [disk\_labels](#input\_disk\_labels) | Labels to be assigned to boot disk, provided as a map | `map(string)` | `{}` | no | +| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `string` | `"100"` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either pd-ssd, local-ssd, or pd-standard | `string` | `"pd-standard"` | no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Whether to enable the Confidential VM configuration on the instance. Note that the instance image must support Confidential VMs. See https://cloud.google.com/compute/docs/images | `bool` | `false` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Whether to enable the Shielded VM configuration on the instance. Note that the instance image must support Shielded VMs. See https://cloud.google.com/compute/docs/images | `bool` | `false` | no | +| [gpu](#input\_gpu) | GPU information. Type and count of GPU to attach to the instance template. See https://cloud.google.com/compute/docs/gpus more details | 
object({
    type  = string
    count = number
  })
| `null` | no | +| [instance\_termination\_action](#input\_instance\_termination\_action) | Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `string` | `null` | no | +| [ipv6\_access\_config](#input\_ipv6\_access\_config) | IPv6 access configurations. Currently a max of 1 IPv6 access configuration is supported. If not specified, the instance will have no external IPv6 Internet access. | 
list(object({
    network_tier = string
  }))
| `[]` | no | +| [labels](#input\_labels) | Labels, provided as a map | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Machine type to create, e.g. n1-standard-1 | `string` | `"n1-standard-1"` | no | +| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list: https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | +| [name\_prefix](#input\_name\_prefix) | Name prefix for the instance template | `string` | n/a | yes | +| [network](#input\_network) | The name or self\_link of the network to attach this interface to. Use network attribute for Legacy or Auto subnetted networks and subnetwork for custom subnetted networks. | `string` | `""` | no | +| [network\_ip](#input\_network\_ip) | Private IP address to assign to the instance if desired. | `string` | `""` | no | +| [nic\_type](#input\_nic\_type) | The type of vNIC to be used on this interface. Possible values: GVNIC, VIRTIO\_NET. | `string` | `null` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy | `string` | `"MIGRATE"` | no | +| [preemptible](#input\_preemptible) | Allow the instance to be preempted | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | The GCP project ID | `string` | `null` | no | +| [provisioning\_model](#input\_provisioning\_model) | The provisioning model of the instance | `string` | `null` | no | +| [region](#input\_region) | Region where the instance template should be created. | `string` | n/a | yes | +| [reservation\_affinity](#input\_reservation\_affinity) | Specifies the reservations that this instance can consume from. | `object({ type = string })` | `null` | no | +| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [service\_account](#input\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#service_account. | 
object({
    email  = optional(string)
    scopes = set(string)
  })
| n/a | yes | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Not used unless enable\_shielded\_vm is true. Shielded VM configuration for the instance. | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [source\_image](#input\_source\_image) | Source disk image. If neither source\_image nor source\_image\_family is specified, defaults to the latest public CentOS image. | `string` | `""` | no | +| [source\_image\_family](#input\_source\_image\_family) | Source image family. If neither source\_image nor source\_image\_family is specified, defaults to the latest public CentOS image. | `string` | `"centos-7"` | no | +| [source\_image\_project](#input\_source\_image\_project) | Project where the source image comes from. The default project contains CentOS images. | `string` | `"centos-cloud"` | no | +| [spot](#input\_spot) | Provision as a SPOT preemptible instance.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `bool` | `false` | no | +| [stack\_type](#input\_stack\_type) | The stack type for this network interface to identify whether the IPv6 feature is enabled or not. Values are `IPV4_IPV6` or `IPV4_ONLY`. Default behavior is equivalent to IPV4\_ONLY. | `string` | `null` | no | +| [startup\_script](#input\_startup\_script) | User startup script to run when instances spin up | `string` | `""` | no | +| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach this interface to. The subnetwork must exist in the same region this instance will be created in. Either network or subnetwork must be provided. | `string` | `""` | no | +| [subnetwork\_project](#input\_subnetwork\_project) | The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used. | `string` | `null` | no | +| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | +| [total\_egress\_bandwidth\_tier](#input\_total\_egress\_bandwidth\_tier) | Network bandwidth tier. Note: machine\_type must be a supported type. Values are 'TIER\_1' or 'DEFAULT'.
See https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration for details. | `string` | `"DEFAULT"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [name](#output\_name) | Name of instance template | +| [self\_link](#output\_self\_link) | Self-link of instance template | +| [service\_account](#output\_service\_account) | value | +| [tags](#output\_tags) | Tags that will be associated with instance(s) | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf new file mode 100644 index 0000000000..cf69a3f926 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf @@ -0,0 +1,227 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +######### +# Locals +######### + +locals { + source_image = var.source_image != "" ? var.source_image : "centos-7-v20201112" + source_image_family = var.source_image_family != "" ? var.source_image_family : "centos-7" + source_image_project = var.source_image_project != "" ? var.source_image_project : "centos-cloud" + + boot_disk = [ + { + source_image = var.source_image != "" ? format("${local.source_image_project}/${local.source_image}") : format("${local.source_image_project}/${local.source_image_family}") + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + disk_labels = var.disk_labels + auto_delete = var.auto_delete + disk_resource_manager_tags = var.disk_resource_manager_tags + boot = "true" + }, + ] + + all_disks = concat(local.boot_disk, var.additional_disks) + + # NOTE: Even if all the shielded_instance_config or confidential_instance_config + # values are false, if the config block exists and an unsupported image is chosen, + # the apply will fail so we use a single-value array with the default value to + # initialize the block only if it is enabled. + shielded_vm_configs = var.enable_shielded_vm ? [true] : [] + + gpu_enabled = var.gpu != null + alias_ip_range_enabled = var.alias_ip_range != null + preemptible = var.preemptible || var.spot + on_host_maintenance = ( + local.preemptible || var.enable_confidential_vm || local.gpu_enabled + ? "TERMINATE" + : var.on_host_maintenance + ) + automatic_restart = ( + # must be false when preemptible is true + local.preemptible ? false : var.automatic_restart + ) + + nic_type = var.total_egress_bandwidth_tier == "TIER_1" ? "GVNIC" : var.nic_type + + + provisioning_model = coalesce(var.provisioning_model, local.preemptible ? "SPOT" : "STANDARD") +} + +data "google_project" "this" { + project_id = var.project_id +} + +#################### +# Instance Template +#################### +resource "google_compute_instance_template" "tpl" { + provider = google-beta + name_prefix = "${var.name_prefix}-" + project = var.project_id + machine_type = var.machine_type + labels = var.labels + metadata = var.metadata + tags = var.tags + can_ip_forward = var.can_ip_forward + metadata_startup_script = var.startup_script + region = var.region + min_cpu_platform = var.min_cpu_platform + resource_manager_tags = var.resource_manager_tags + + service_account { + email = coalesce(var.service_account.email, "${data.google_project.this.number}-compute@developer.gserviceaccount.com") + scopes = lookup(var.service_account, "scopes", null) + } + + dynamic "disk" { + for_each = local.all_disks + content { + auto_delete = lookup(disk.value, "auto_delete", null) + boot = lookup(disk.value, "boot", null) + device_name = lookup(disk.value, "device_name", null) + disk_name = lookup(disk.value, "disk_name", null) + disk_size_gb = lookup(disk.value, "disk_size_gb", lookup(disk.value, "disk_type", null) == "local-ssd" ? "375" : null) + disk_type = lookup(disk.value, "disk_type", null) + interface = lookup(disk.value, "interface", lookup(disk.value, "disk_type", null) == "local-ssd" ? "NVME" : null) + mode = lookup(disk.value, "mode", null) + source = lookup(disk.value, "source", null) + source_image = lookup(disk.value, "source_image", null) + type = lookup(disk.value, "disk_type", null) == "local-ssd" ? "SCRATCH" : "PERSISTENT" + labels = (lookup(disk.value, "source", null) != null || lookup(disk.value, "disk_type", null) == "local-ssd") ? null : lookup(disk.value, "disk_labels", null) + resource_manager_tags = lookup(disk.value, "disk_resource_manager_tags", {}) + + dynamic "disk_encryption_key" { + for_each = compact([var.disk_encryption_key == null ? null : 1]) + content { + kms_key_self_link = var.disk_encryption_key + } + } + } + } + + network_interface { + network = var.network + subnetwork = var.subnetwork + subnetwork_project = var.subnetwork_project + network_ip = try(coalesce(var.network_ip), null) + nic_type = local.nic_type + stack_type = var.stack_type + dynamic "access_config" { + for_each = var.access_config + content { + nat_ip = access_config.value.nat_ip + network_tier = access_config.value.network_tier + } + } + dynamic "ipv6_access_config" { + for_each = var.ipv6_access_config + content { + network_tier = ipv6_access_config.value.network_tier + } + } + dynamic "alias_ip_range" { + for_each = local.alias_ip_range_enabled ? [var.alias_ip_range] : [] + content { + ip_cidr_range = alias_ip_range.value.ip_cidr_range + subnetwork_range_name = alias_ip_range.value.subnetwork_range_name + } + } + } + + dynamic "network_interface" { + for_each = var.additional_networks + content { + network = network_interface.value.network + subnetwork = network_interface.value.subnetwork + subnetwork_project = network_interface.value.subnetwork_project + network_ip = try(coalesce(network_interface.value.network_ip), null) + nic_type = try(coalesce(network_interface.value.nic_type), null) + dynamic "access_config" { + for_each = network_interface.value.access_config + content { + nat_ip = access_config.value.nat_ip + network_tier = access_config.value.network_tier + } + } + dynamic "ipv6_access_config" { + for_each = network_interface.value.ipv6_access_config + content { + network_tier = ipv6_access_config.value.network_tier + } + } + } + } + + network_performance_config { + total_egress_bandwidth_tier = coalesce(var.total_egress_bandwidth_tier, "DEFAULT") + } + + lifecycle { + create_before_destroy = "true" + } + + scheduling { + preemptible = local.preemptible + provisioning_model = local.provisioning_model + automatic_restart = local.automatic_restart + on_host_maintenance = local.on_host_maintenance + instance_termination_action = var.instance_termination_action + + dynamic "max_run_duration" { + for_each = var.max_run_duration != null ? [var.max_run_duration] : [] + content { + seconds = max_run_duration.value + } + } + } + + dynamic "reservation_affinity" { + for_each = var.reservation_affinity != null ? [var.reservation_affinity] : [] + content { + type = reservation_affinity.value.type + } + } + + advanced_machine_features { + enable_nested_virtualization = var.advanced_machine_features.enable_nested_virtualization + threads_per_core = var.advanced_machine_features.threads_per_core + turbo_mode = var.advanced_machine_features.turbo_mode + visible_core_count = var.advanced_machine_features.visible_core_count + performance_monitoring_unit = var.advanced_machine_features.performance_monitoring_unit + enable_uefi_networking = var.advanced_machine_features.enable_uefi_networking + } + + dynamic "shielded_instance_config" { + for_each = local.shielded_vm_configs + content { + enable_secure_boot = lookup(var.shielded_instance_config, "enable_secure_boot", shielded_instance_config.value) + enable_vtpm = lookup(var.shielded_instance_config, "enable_vtpm", shielded_instance_config.value) + enable_integrity_monitoring = lookup(var.shielded_instance_config, "enable_integrity_monitoring", shielded_instance_config.value) + } + } + + confidential_instance_config { + enable_confidential_compute = var.enable_confidential_vm + } + + dynamic "guest_accelerator" { + for_each = local.gpu_enabled ? [var.gpu] : [] + content { + type = guest_accelerator.value.type + count = guest_accelerator.value.count + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf new file mode 100644 index 0000000000..69f8d3b98c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf @@ -0,0 +1,33 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "self_link" { + description = "Self-link of instance template" + value = google_compute_instance_template.tpl.self_link +} + +output "name" { + description = "Name of instance template" + value = google_compute_instance_template.tpl.name +} + +output "tags" { + description = "Tags that will be associated with instance(s)" + value = google_compute_instance_template.tpl.tags +} + +output "service_account" { + description = "value" + value = google_compute_instance_template.tpl.service_account[0] +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf new file mode 100644 index 0000000000..c285c3fea5 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf @@ -0,0 +1,398 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + type = string + description = "The GCP project ID" + default = null +} + +variable "name_prefix" { + description = "Name prefix for the instance template" + type = string +} + +variable "machine_type" { + description = "Machine type to create, e.g. n1-standard-1" + type = string + default = "n1-standard-1" +} + +variable "min_cpu_platform" { + description = "Specifies a minimum CPU platform. Applicable values are the friendly names of CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list: https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform" + type = string + default = null +} + +variable "can_ip_forward" { + description = "Enable IP forwarding, for NAT instances for example" + type = string + default = "false" +} + +variable "tags" { + type = list(string) + description = "Network tags, provided as a list" + default = [] +} + +variable "labels" { + type = map(string) + description = "Labels, provided as a map" + default = {} +} + +variable "preemptible" { + type = bool + description = "Allow the instance to be preempted" + default = false +} + +variable "spot" { + description = <<-EOD + Provision as a SPOT preemptible instance. + See https://cloud.google.com/compute/docs/instances/spot for more details. + EOD + type = bool + default = false +} + +variable "instance_termination_action" { + description = <<-EOD + Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'. + See https://cloud.google.com/compute/docs/instances/spot for more details. + EOD + type = string + default = null +} + +variable "automatic_restart" { + type = bool + description = "(Optional) Specifies whether the instance should be automatically restarted if it is terminated by Compute Engine (not terminated by a user)." + default = true +} + +variable "on_host_maintenance" { + type = string + description = "Instance availability Policy" + default = "MIGRATE" +} + +variable "region" { + type = string + description = "Region where the instance template should be created." + nullable = false +} + +variable "advanced_machine_features" { + description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" + type = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) +} + +variable "resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} + validation { + condition = alltrue([for value in var.resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) + error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" + } + validation { + condition = alltrue([for value in keys(var.resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) + error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" + } +} + +####### +# disk +####### +variable "source_image" { + description = "Source disk image. If neither source_image nor source_image_family is specified, defaults to the latest public CentOS image." + type = string + default = "" +} + +variable "source_image_family" { + description = "Source image family. If neither source_image nor source_image_family is specified, defaults to the latest public CentOS image." + type = string + default = "centos-7" +} + +variable "source_image_project" { + description = "Project where the source image comes from. The default project contains CentOS images." + type = string + default = "centos-cloud" +} + +variable "disk_size_gb" { + description = "Boot disk size in GB" + type = string + default = "100" +} + +variable "disk_type" { + description = "Boot disk type, can be either pd-ssd, local-ssd, or pd-standard" + type = string + default = "pd-standard" +} + +variable "disk_labels" { + description = "Labels to be assigned to boot disk, provided as a map" + type = map(string) + default = {} +} + +variable "disk_encryption_key" { + description = "The id of the encryption key that is stored in Google Cloud KMS to use to encrypt all the disks on this instance" + type = string + default = null +} + +variable "auto_delete" { + description = "Whether or not the boot disk should be auto-deleted" + type = string + default = "true" +} + +variable "disk_resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} + validation { + condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) + error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" + } + validation { + condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) + error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" + } +} + +variable "additional_disks" { + description = "List of maps of additional disks. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#disk_name" + type = list(object({ + source = optional(string) + disk_name = optional(string) + device_name = string + auto_delete = bool + boot = bool + disk_size_gb = optional(number) + disk_type = optional(string) + disk_labels = map(string) + disk_resource_manager_tags = map(string) + })) + default = [] +} + +#################### +# network_interface +#################### +variable "network" { + description = "The name or self_link of the network to attach this interface to. Use network attribute for Legacy or Auto subnetted networks and subnetwork for custom subnetted networks." + type = string + default = "" +} + +variable "nic_type" { + description = "The type of vNIC to be used on this interface. Possible values: GVNIC, VIRTIO_NET." + type = string + default = null +} + +variable "subnetwork" { + description = "The name of the subnetwork to attach this interface to. The subnetwork must exist in the same region this instance will be created in. Either network or subnetwork must be provided." + type = string + default = "" +} + +variable "subnetwork_project" { + description = "The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used." + type = string + default = null +} + +variable "network_ip" { + description = "Private IP address to assign to the instance if desired." + type = string + default = "" +} + +variable "stack_type" { + description = "The stack type for this network interface to identify whether the IPv6 feature is enabled or not. Values are `IPV4_IPV6` or `IPV4_ONLY`. Default behavior is equivalent to IPV4_ONLY." + type = string + default = null +} + +variable "additional_networks" { + description = "Additional network interface details for GCE, if any." + default = [] + type = list(object({ + network = string + subnetwork = string + subnetwork_project = string + network_ip = string + nic_type = string + access_config = list(object({ + nat_ip = string + network_tier = string + })) + ipv6_access_config = list(object({ + network_tier = string + })) + })) +} + +variable "total_egress_bandwidth_tier" { + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 4.84 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.84 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [instance](#module\_instance) | ../instance | n/a | +| [template](#module\_template) | ../instance_template | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [internal\_startup\_script](#input\_internal\_startup\_script) | FOR INTERNAL TOOLKIT USAGE ONLY. | `string` | `null` | no | +| [login\_nodes](#input\_login\_nodes) | Slurm login instance definitions. | 
object({
    group_name = string
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    additional_networks = optional(list(object({
      access_config = optional(list(object({
        nat_ip       = string
        network_tier = string
      })), [])
      alias_ip_range = optional(list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      })), [])
      ipv6_access_config = optional(list(object({
        network_tier = string
      })), [])
      network            = optional(string)
      network_ip         = optional(string, "")
      nic_type           = optional(string)
      queue_count        = optional(number)
      stack_type         = optional(string)
      subnetwork         = optional(string)
      subnetwork_project = optional(string)
    })), [])
    bandwidth_tier             = optional(string, "platform_default")
    can_ip_forward             = optional(bool, false)
    disk_auto_delete           = optional(bool, true)
    disk_labels                = optional(map(string), {})
    disk_resource_manager_tags = optional(map(string), {})
    disk_size_gb               = optional(number)
    disk_type                  = optional(string, "n1-standard-1")
    enable_confidential_vm     = optional(bool, false)
    enable_oslogin             = optional(bool, true)
    enable_shielded_vm         = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    metadata              = optional(map(string), {})
    min_cpu_platform      = optional(string)
    num_instances         = optional(number, 1)
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    static_ips           = optional(list(string), [])
    subnetwork           = string
    spot                 = optional(bool, false)
    tags                 = optional(list(string), [])
    zone                 = optional(string)
    termination_action   = optional(string)
  })
| n/a | yes | +| [network\_storage](#input\_network\_storage) | Storage to mounted on login instances
- server\_ip : Address of the storage server.
- remote\_mount : The location in the remote instance filesystem to mount from.
- local\_mount : The location on the instance filesystem to mount to.
- fs\_type : Filesystem type (e.g. "nfs").
- mount\_options : Options to mount with. | 
list(object({
    server_ip     = string
    remote_mount  = string
    local_mount   = string
    fs_type       = string
    mount_options = string
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [replace\_trigger](#input\_replace\_trigger) | Trigger value to replace the instances. | `string` | `""` | no | +| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path to directory in the bucket for configs | `string` | n/a | yes | +| [slurm\_bucket\_name](#input\_slurm\_bucket\_name) | Name of the bucket for configs | `string` | n/a | yes | +| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name | `string` | n/a | yes | +| [startup\_scripts](#input\_startup\_scripts) | List of scripts to be ran on login VMs startup. | 
list(object({
    filename = string
    content  = string
  }))
| `[]` | no | +| [startup\_scripts\_timeout](#input\_startup\_scripts\_timeout) | The timeout (seconds) applied to each startup script. If any script exceeds this timeout,
then the instance setup process is considered failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instances](#output\_instances) | VM instances of login nodes | +| [service\_account](#output\_service\_account) | Service Account used by login VMs | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf new file mode 100644 index 0000000000..8185f71fd6 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf @@ -0,0 +1,110 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module "template" { + source = "../instance_template" + + project_id = var.project_id + slurm_cluster_name = var.slurm_cluster_name + slurm_instance_role = "login" + slurm_bucket_path = var.slurm_bucket_path + name_prefix = local.name + + additional_disks = var.login_nodes.additional_disks + bandwidth_tier = var.login_nodes.bandwidth_tier + can_ip_forward = var.login_nodes.can_ip_forward + advanced_machine_features = var.login_nodes.advanced_machine_features + disk_auto_delete = var.login_nodes.disk_auto_delete + disk_labels = var.login_nodes.disk_labels + disk_resource_manager_tags = var.login_nodes.disk_resource_manager_tags + disk_size_gb = var.login_nodes.disk_size_gb + disk_type = var.login_nodes.disk_type + enable_confidential_vm = var.login_nodes.enable_confidential_vm + enable_oslogin = var.login_nodes.enable_oslogin + enable_shielded_vm = var.login_nodes.enable_shielded_vm + gpu = var.login_nodes.gpu + labels = var.login_nodes.labels + machine_type = var.login_nodes.machine_type + metadata = merge(var.login_nodes.metadata, { + "universe_domain" = var.universe_domain, + "slurm_login_group" = local.name + }) + min_cpu_platform = var.login_nodes.min_cpu_platform + on_host_maintenance = var.login_nodes.on_host_maintenance + preemptible = var.login_nodes.preemptible + region = var.login_nodes.region + resource_manager_tags = var.login_nodes.resource_manager_tags + service_account = var.login_nodes.service_account + shielded_instance_config = var.login_nodes.shielded_instance_config + source_image_family = var.login_nodes.source_image_family + source_image_project = var.login_nodes.source_image_project + source_image = var.login_nodes.source_image + spot = var.login_nodes.spot + subnetwork = var.login_nodes.subnetwork + tags = concat([var.slurm_cluster_name], var.login_nodes.tags) + termination_action = var.login_nodes.termination_action + + internal_startup_script = var.internal_startup_script +} + +module "instance" { + source = "../instance" + + access_config = var.login_nodes.access_config + hostname = "${var.slurm_cluster_name}-${local.name}" + + project_id = var.project_id + + instance_template = module.template.self_link + num_instances = var.login_nodes.num_instances + + additional_networks = var.login_nodes.additional_networks + region = var.login_nodes.region + static_ips = var.login_nodes.static_ips + subnetwork = var.login_nodes.subnetwork + zone = var.login_nodes.zone + + replace_trigger = var.replace_trigger +} + +resource "google_storage_bucket_object" "startup_scripts" { + for_each = { + for s in var.startup_scripts : format( + "slurm-login-%s-script-%s", local.name, replace(basename(s.filename), "/[^a-zA-Z0-9-_]/", "_") + ) => s.content + } + + bucket = var.slurm_bucket_name + name = "${var.slurm_bucket_dir}/${each.key}" + content = each.value +} + +locals { + name = var.login_nodes.group_name # short hand + + config = { + group_name = local.name + startup_scripts_timeout = var.startup_scripts_timeout + network_storage = var.network_storage + } +} + +resource "google_storage_bucket_object" "config" { + bucket = var.slurm_bucket_name + name = "${var.slurm_bucket_dir}/login_group_configs/${local.name}.yaml" + content = yamlencode(local.config) + + # To ensure that login group "is not ready" until all startup scripts are written down + depends_on = [google_storage_bucket_object.startup_scripts] +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf new file mode 100644 index 0000000000..04de18a188 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "service_account" { + value = module.template.service_account + description = "Service Account used by login VMs" +} + +output "instances" { + value = module.instance.slurm_instances + description = "VM instances of login nodes" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf new file mode 100644 index 0000000000..3efd862942 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf @@ -0,0 +1,188 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "slurm_cluster_name" { + type = string + description = "Cluster name" +} + +variable "slurm_bucket_path" { + type = string + description = "GCS Bucket URI of Slurm cluster file storage." +} + + +variable "slurm_bucket_name" { + type = string + description = "Name of the bucket for configs" +} + +variable "slurm_bucket_dir" { + type = string + description = "Path to directory in the bucket for configs" +} + + +variable "universe_domain" { + description = "Domain address for alternate API universe" + type = string + default = "googleapis.com" +} + +variable "login_nodes" { + description = "Slurm login instance definitions." + type = object({ + group_name = string + access_config = optional(list(object({ + nat_ip = string + network_tier = string + }))) + additional_disks = optional(list(object({ + disk_name = optional(string) + device_name = optional(string) + disk_size_gb = optional(number) + disk_type = optional(string) + disk_labels = optional(map(string), {}) + auto_delete = optional(bool, true) + boot = optional(bool, false) + disk_resource_manager_tags = optional(map(string), {}) + })), []) + additional_networks = optional(list(object({ + access_config = optional(list(object({ + nat_ip = string + network_tier = string + })), []) + alias_ip_range = optional(list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })), []) + ipv6_access_config = optional(list(object({ + network_tier = string + })), []) + network = optional(string) + network_ip = optional(string, "") + nic_type = optional(string) + queue_count = optional(number) + stack_type = optional(string) + subnetwork = optional(string) + subnetwork_project = optional(string) + })), []) + bandwidth_tier = optional(string, "platform_default") + can_ip_forward = optional(bool, false) + disk_auto_delete = optional(bool, true) + disk_labels = optional(map(string), {}) + disk_resource_manager_tags = optional(map(string), {}) + disk_size_gb = optional(number) + disk_type = optional(string, "n1-standard-1") + enable_confidential_vm = optional(bool, false) + enable_oslogin = optional(bool, true) + enable_shielded_vm = optional(bool, false) + gpu = optional(object({ + count = number + type = string + })) + labels = optional(map(string), {}) + machine_type = optional(string) + advanced_machine_features = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) + metadata = optional(map(string), {}) + min_cpu_platform = optional(string) + num_instances = optional(number, 1) + on_host_maintenance = optional(string) + preemptible = optional(bool, false) + region = optional(string) + resource_manager_tags = optional(map(string), {}) + service_account = optional(object({ + email = optional(string) + scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) + })) + shielded_instance_config = optional(object({ + enable_integrity_monitoring = optional(bool, true) + enable_secure_boot = optional(bool, true) + enable_vtpm = optional(bool, true) + })) + source_image_family = optional(string) + source_image_project = optional(string) + source_image = optional(string) + static_ips = optional(list(string), []) + subnetwork = string + spot = optional(bool, false) + tags = optional(list(string), []) + zone = optional(string) + termination_action = optional(string) + }) +} + + +variable "startup_scripts" { + description = "List of scripts to be ran on login VMs startup." + type = list(object({ + filename = string + content = string + })) + default = [] +} + +variable "startup_scripts_timeout" { + description = < + +- [Module: Slurm Nodeset (TPU)](#module-slurm-nodeset-tpu) + - [Overview](#overview) + - [Module API](#module-api) + + + +## Overview + +This is a submodule of [slurm_cluster](../../../slurm_cluster/README.md). It +creates a Slurm TPU nodeset for [slurm_partition](../slurm_partition/README.md). + +## Module API + +For the terraform module API reference, please see +[README_TF.md](./README_TF.md). + + +Copyright (C) SchedMD LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~> 1.2 | +| [google](#requirement\_google) | >= 3.53 | +| [null](#requirement\_null) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.53 | +| [null](#provider\_null) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.nodeset_tpu](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_subnetwork.nodeset_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [accelerator\_config](#input\_accelerator\_config) | Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details. | 
object({
    topology = string
    version  = string
  })
| 
{
  "topology": "",
  "version": ""
}
| no | +| [data\_disks](#input\_data\_disks) | The data disks to include in the TPU node | `list(string)` | `[]` | no | +| [docker\_image](#input\_docker\_image) | The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf- | `string` | `""` | no | +| [enable\_public\_ip](#input\_enable\_public\_ip) | Enables IP address to access the Internet. | `bool` | `false` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | +| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of nodes allowed in this partition to be created dynamically. | `number` | `0` | no | +| [node\_count\_static](#input\_node\_count\_static) | Number of nodes to be statically created. | `number` | `0` | no | +| [node\_type](#input\_node\_type) | Specify a node type to base the vm configuration upon it. Not needed if you use accelerator\_config | `string` | `null` | no | +| [nodeset\_name](#input\_nodeset\_name) | Name of Slurm nodeset. | `string` | n/a | yes | +| [preemptible](#input\_preemptible) | Specify whether TPU-vms in this nodeset are preemtible, see https://cloud.google.com/tpu/docs/preemptible for details. | `bool` | `false` | no | +| [preserve\_tpu](#input\_preserve\_tpu) | Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted | `bool` | `true` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [reserved](#input\_reserved) | Specify whether TPU-vms in this nodeset are created under a reservation. | `bool` | `false` | no | +| [service\_account](#input\_service\_account) | Service account to attach to the TPU-vm.
If none is given, the default service account and scopes will be used. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach the TPU-vm of this nodeset to. | `string` | n/a | yes | +| [tf\_version](#input\_tf\_version) | Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details. | `string` | n/a | yes | +| [zone](#input\_zone) | Nodes will only be created in this zone. Check https://cloud.google.com/tpu/docs/regions-zones to get zones with TPU-vm in it. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [nodeset](#output\_nodeset) | Nodeset details. | +| [nodeset\_name](#output\_nodeset\_name) | Nodeset name. | +| [service\_account](#output\_service\_account) | Service account object, includes email and scopes. | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf new file mode 100644 index 0000000000..1a6a9cfba1 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf @@ -0,0 +1,121 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +########### +# NODESET # +########### + +locals { + node_conf_hw = { + Mem334CPU96 = { + CPUs = 96 + Boards = 1 + Sockets = 2 + CoresPerSocket = 24 + ThreadsPerCore = 2 + RealMemory = 307200 + } + Mem400CPU240 = { + CPUs = 240 + Boards = 1 + Sockets = 2 + CoresPerSocket = 60 + ThreadsPerCore = 2 + RealMemory = 400000 + } + } + node_conf_mappings = { + "v2" = local.node_conf_hw.Mem334CPU96 + "v3" = local.node_conf_hw.Mem334CPU96 + "v4" = local.node_conf_hw.Mem400CPU240 + } + simple_nodes = ["v2-8", "v3-8", "v4-8"] +} + +locals { + snetwork = data.google_compute_subnetwork.nodeset_subnetwork.name + region = join("-", slice(split("-", var.zone), 0, 2)) + tpu_fam = var.accelerator_config.version != "" ? lower(var.accelerator_config.version) : split("-", var.node_type)[0] + #If subnetwork is specified and it does not have private_ip_google_access, we need to have public IPs on the TPU + #if no subnetwork is specified, the default one will be used, this does not have private_ip_google_access so we need public IPs too + pub_need = !data.google_compute_subnetwork.nodeset_subnetwork.private_ip_google_access + can_preempt = var.node_type != null ? contains(local.simple_nodes, var.node_type) : false + nodeset_tpu = { + nodeset_name = var.nodeset_name + node_conf = local.node_conf_mappings[local.tpu_fam] + node_type = var.node_type + accelerator_config = var.accelerator_config + tf_version = var.tf_version + preemptible = local.can_preempt ? var.preemptible : false + reserved = var.reserved + node_count_dynamic_max = var.node_count_dynamic_max + node_count_static = var.node_count_static + enable_public_ip = var.enable_public_ip + zone = var.zone + service_account = var.service_account != null ? var.service_account : local.service_account + preserve_tpu = local.can_preempt ? var.preserve_tpu : false + data_disks = var.data_disks + docker_image = var.docker_image != "" ? var.docker_image : "us-docker.pkg.dev/schedmd-slurm-public/tpu/slurm-gcp-6-9:tf-${var.tf_version}" + subnetwork = local.snetwork + network_storage = var.network_storage + } + + service_account = { + email = try(var.service_account.email, null) + scopes = try(var.service_account.scopes, ["https://www.googleapis.com/auth/cloud-platform"]) + } +} + +data "google_compute_subnetwork" "nodeset_subnetwork" { + name = var.subnetwork + region = local.region + project = var.project_id + + self_link = ( + length(regexall("/projects/([^/]*)", var.subnetwork)) > 0 + && length(regexall("/regions/([^/]*)", var.subnetwork)) > 0 + ? var.subnetwork + : null + ) +} + +resource "null_resource" "nodeset_tpu" { + triggers = { + nodeset = sha256(jsonencode(local.nodeset_tpu)) + } + lifecycle { + precondition { + condition = sum([var.node_count_dynamic_max, var.node_count_static]) > 0 + error_message = "Sum of node_count_dynamic_max and node_count_static must be > 0." + } + precondition { + condition = !(var.preemptible && var.reserved) + error_message = "Nodeset cannot be preemptible and reserved at the same time." + } + precondition { + condition = !(var.subnetwork == null && !var.enable_public_ip) + error_message = "Using the default subnetwork for the TPU nodeset requires enable_public_ip set to true." + } + precondition { + condition = !(var.subnetwork != null && (local.pub_need && !var.enable_public_ip)) + error_message = "The subnetwork specified does not have Private Google Access enabled. This is required when enable_public_ip is set to false." + } + precondition { + condition = !(var.node_type == null && (var.accelerator_config.topology == "" && var.accelerator_config.version == "")) + error_message = "Either a node type or an accelerator_config must be provided." + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf new file mode 100644 index 0000000000..fce700d567 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "nodeset_name" { + description = "Nodeset name." + value = local.nodeset_tpu.nodeset_name +} + +output "nodeset" { + description = "Nodeset details." + value = local.nodeset_tpu +} + +output "service_account" { + description = "Service account object, includes email and scopes." + value = local.service_account +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf new file mode 100644 index 0000000000..a8c470dec9 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf @@ -0,0 +1,158 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "nodeset_name" { + description = "Name of Slurm nodeset." + type = string + + validation { + condition = can(regex("^[a-z](?:[a-z0-9]{0,14})$", var.nodeset_name)) + error_message = "Variable 'nodeset_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,14})$'." + } +} + +variable "node_type" { + description = "Specify a node type to base the vm configuration upon it. Not needed if you use accelerator_config" + type = string + default = null +} + +variable "accelerator_config" { + description = "Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details." + type = object({ + topology = string + version = string + }) + default = { + topology = "" + version = "" + } + validation { + condition = var.accelerator_config.version == "" ? true : contains(["V2", "V3", "V4"], upper(var.accelerator_config.version)) + error_message = "accelerator_config.version must be one of [\"V2\", \"V3\", \"V4\"]" + } + validation { + condition = var.accelerator_config.topology == "" ? true : can(regex("^[1-9]x[1-9](x[1-9])?$", var.accelerator_config.topology)) + error_message = "accelerator_config.topology must be a valid topology, like 2x2 4x4x4 4x2x4 etc..." + } +} + +variable "docker_image" { + description = "The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf-" + type = string + default = "" +} + +variable "tf_version" { + description = "Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details." + type = string +} + +variable "zone" { + description = "Nodes will only be created in this zone. Check https://cloud.google.com/tpu/docs/regions-zones to get zones with TPU-vm in it." + type = string + + validation { + condition = can(coalesce(var.zone)) + error_message = "Zone cannot be null or empty." + } +} + +variable "preemptible" { + description = "Specify whether TPU-vms in this nodeset are preemtible, see https://cloud.google.com/tpu/docs/preemptible for details." + type = bool + default = false +} + +variable "reserved" { + description = "Specify whether TPU-vms in this nodeset are created under a reservation." + type = bool + default = false +} + +variable "preserve_tpu" { + description = "Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted" + type = bool + default = true +} + +variable "node_count_static" { + description = "Number of nodes to be statically created." + type = number + default = 0 + + validation { + condition = var.node_count_static >= 0 + error_message = "Value must be >= 0." + } +} + +variable "node_count_dynamic_max" { + description = "Maximum number of nodes allowed in this partition to be created dynamically." + type = number + default = 0 + + validation { + condition = var.node_count_dynamic_max >= 0 + error_message = "Value must be >= 0." + } +} + +variable "enable_public_ip" { + description = "Enables IP address to access the Internet." + type = bool + default = false +} + +variable "data_disks" { + type = list(string) + description = "The data disks to include in the TPU node" + default = [] +} + +variable "subnetwork" { + description = "The name of the subnetwork to attach the TPU-vm of this nodeset to." + type = string +} + +variable "service_account" { + type = object({ + email = string + scopes = set(string) + }) + description = < +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | > 5.0 | +| [helm](#requirement\_helm) | ~> 2.17 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | > 5.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [install\_gpu\_operator](#module\_install\_gpu\_operator) | ./helm_install | n/a | +| [install\_jobset](#module\_install\_jobset) | ./helm_install | n/a | +| [install\_kueue](#module\_install\_kueue) | ./helm_install | n/a | +| [install\_nvidia\_dra\_driver](#module\_install\_nvidia\_dra\_driver) | ./helm_install | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | +| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cluster\_id](#input\_cluster\_id) | An identifier for the gke cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | +| [gke\_cluster\_exists](#input\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | `bool` | `false` | no | +| [gpu\_operator](#input\_gpu\_operator) | Install [GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) which uses the [Kubernetes operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) to automate the management of all NVIDIA software components needed to provision GPU. | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | +| [jobset](#input\_jobset) | Install [Jobset](https://github.com/kubernetes-sigs/jobset) which manages a group of K8s [jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) as a unit. | 
object({
    install = optional(bool, false)
    version = optional(string, "v0.7.2")
  })
| `{}` | no | +| [kueue](#input\_kueue) | Install and configure [Kueue](https://kueue.sigs.k8s.io/docs/overview/) workload scheduler. A configuration yaml/template file can be provided with config\_path to be applied right after kueue installation. If a template file provided, its variables can be set to config\_template\_vars. | 
object({
    install              = optional(bool, false)
    version              = optional(string, "v0.11.4")
    config_path          = optional(string, null)
    config_template_vars = optional(map(any), null)
  })
| `{}` | no | +| [nvidia\_dra\_driver](#input\_nvidia\_dra\_driver) | Installs [Nvidia DRA driver](https://github.com/NVIDIA/k8s-dra-driver-gpu) which supports Dynamic Resource Allocation for NVIDIA GPUs in Kubernetes | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0-rc.2")
  })
| `{}` | no | +| [project\_id](#input\_project\_id) | The project ID that hosts the gke cluster. | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md new file mode 100644 index 0000000000..1957899617 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md @@ -0,0 +1,64 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [helm](#requirement\_helm) | ~> 2.17 | + +## Providers + +| Name | Version | +|------|---------| +| [helm](#provider\_helm) | ~> 2.17 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [helm_release.apply_chart](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [atomic](#input\_atomic) | If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used. | `bool` | `false` | no | +| [chart\_name](#input\_chart\_name) | Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL). | `string` | n/a | yes | +| [chart\_repository](#input\_chart\_repository) | URL of the Helm chart repository. Set to null or omit if 'chart\_name' is a path or URL. | `string` | `null` | no | +| [chart\_version](#input\_chart\_version) | Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true). | `string` | `null` | no | +| [cleanup\_on\_fail](#input\_cleanup\_on\_fail) | Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail'). | `bool` | `false` | no | +| [create\_namespace](#input\_create\_namespace) | Set to true to create the namespace if it does not exist ('helm install --create-namespace'). | `bool` | `true` | no | +| [dependency\_update](#input\_dependency\_update) | Run 'helm dependency update' before installing the chart (useful if chart\_name is a local path to an unpacked chart with dependencies). | `bool` | `false` | no | +| [description](#input\_description) | Set an optional description for the Helm release. | `string` | `null` | no | +| [devel](#input\_devel) | Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart\_version' is set, this is ignored. | `bool` | `false` | no | +| [disable\_crd\_hooks](#input\_disable\_crd\_hooks) | Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook'). | `bool` | `false` | no | +| [disable\_openapi\_validation](#input\_disable\_openapi\_validation) | If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation'). | `bool` | `false` | no | +| [disable\_webhooks](#input\_disable\_webhooks) | Prevent hooks from running ('helm install --no-hooks'). | `bool` | `false` | no | +| [force\_update](#input\_force\_update) | Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution. | `bool` | `false` | no | +| [keyring](#input\_keyring) | Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true. | `string` | `null` | no | +| [lint](#input\_lint) | Run the helm chart linter during the plan ('helm lint'). | `bool` | `false` | no | +| [max\_history](#input\_max\_history) | Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit. | `number` | `null` | no | +| [namespace](#input\_namespace) | Kubernetes namespace to install the Helm release into. | `string` | `"default"` | no | +| [pass\_credentials](#input\_pass\_credentials) | Pass credentials to all domains ('helm install --pass-credentials'). Use with caution. | `bool` | `false` | no | +| [postrender](#input\_postrender) | Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary\_path' attribute. | 
object({
    binary_path = string # Path to the post-renderer executable
  })
| `null` | no | +| [recreate\_pods](#input\_recreate\_pods) | Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself. | `bool` | `false` | no | +| [release\_name](#input\_release\_name) | Name of the Helm release. | `string` | n/a | yes | +| [render\_subchart\_notes](#input\_render\_subchart\_notes) | If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes'). | `bool` | `false` | no | +| [reset\_values](#input\_reset\_values) | When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values'). | `bool` | `false` | no | +| [reuse\_values](#input\_reuse\_values) | When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset\_values' is specified, this is ignored. | `bool` | `false` | no | +| [set\_values](#input\_set\_values) | List of objects defining values to set ('helm install --set'). | 
list(object({
    name  = string                     # Path to the value (e.g., 'service.type', 'replicaCount')
    value = string                     # The value to set
    type  = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file')
  }))
| `[]` | no | +| [skip\_crds](#input\_skip\_crds) | If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present. | `bool` | `false` | no | +| [timeout](#input\_timeout) | Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout'). | `number` | `300` | no | +| [values\_yaml](#input\_values\_yaml) | List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile(). | `list(string)` | `[]` | no | +| [verify](#input\_verify) | Verify the package before installing it ('helm install --verify'). | `bool` | `false` | no | +| [wait](#input\_wait) | Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait'). | `bool` | `true` | no | +| [wait\_for\_jobs](#input\_wait\_for\_jobs) | If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs'). | `bool` | `false` | no | + +## Outputs + +No outputs. + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf new file mode 100644 index 0000000000..bd2383b772 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf @@ -0,0 +1,75 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "helm_release" "apply_chart" { + # Required Identification + name = var.release_name + chart = var.chart_name + + # Chart Source & Version + repository = var.chart_repository + version = var.chart_version + devel = var.devel + + # Target Namespace + namespace = var.namespace + create_namespace = var.create_namespace + + # Values Configuration + values = var.values_yaml + + dynamic "set" { + for_each = var.set_values + content { + name = set.value.name + value = set.value.value + type = set.value.type + } + } + + # Installation/Upgrade Behavior + description = var.description + atomic = var.atomic + cleanup_on_fail = var.cleanup_on_fail + dependency_update = var.dependency_update + disable_crd_hooks = var.disable_crd_hooks + disable_openapi_validation = var.disable_openapi_validation + disable_webhooks = var.disable_webhooks + force_update = var.force_update + lint = var.lint + max_history = var.max_history + recreate_pods = var.recreate_pods # Note: Deprecated in Helm CLI + render_subchart_notes = var.render_subchart_notes + reset_values = var.reset_values + reuse_values = var.reuse_values + skip_crds = var.skip_crds + timeout = var.timeout + wait = var.wait + wait_for_jobs = var.wait_for_jobs + + # Verification & Credentials + keyring = var.keyring + pass_credentials = var.pass_credentials + verify = var.verify + + # Post Rendering + dynamic "postrender" { + # Only include the block if var.postrender is not null + for_each = var.postrender == null ? [] : [var.postrender] + content { + binary_path = postrender.value.binary_path + } + } + +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml new file mode 100644 index 0000000000..e18197e2b7 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf new file mode 100644 index 0000000000..04e8e214fc --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf @@ -0,0 +1,212 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Description: Input variables for the generic Helm release module. + +# --- Required --- +variable "release_name" { + description = "Name of the Helm release." + type = string +} + +variable "chart_name" { + description = "Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL)." + type = string +} + +# --- Chart Location & Version --- +variable "chart_repository" { + description = "URL of the Helm chart repository. Set to null or omit if 'chart_name' is a path or URL." + type = string + default = null +} + +variable "chart_version" { + description = "Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true)." + type = string + default = null +} + +variable "devel" { + description = "Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart_version' is set, this is ignored." + type = bool + default = false +} + +# --- Namespace --- +variable "namespace" { + description = "Kubernetes namespace to install the Helm release into." + type = string + default = "default" +} + +variable "create_namespace" { + description = "Set to true to create the namespace if it does not exist ('helm install --create-namespace')." + type = bool + default = true # Common convenience setting +} + +# --- Values Customization --- +variable "values_yaml" { + description = "List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile()." + type = list(string) + default = [] +} + +variable "set_values" { + description = "List of objects defining values to set ('helm install --set')." + type = list(object({ + name = string # Path to the value (e.g., 'service.type', 'replicaCount') + value = string # The value to set + type = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file') + })) + default = [] +} + +# --- Installation/Upgrade Behavior --- +variable "description" { + description = "Set an optional description for the Helm release." + type = string + default = null +} + +variable "atomic" { + description = "If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used." + type = bool + default = false +} + +variable "wait" { + description = "Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait')." + type = bool + default = true # Often a good default for dependencies +} + +variable "wait_for_jobs" { + description = "If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs')." + type = bool + default = false # Helm CLI default is false +} + +variable "timeout" { + description = "Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout')." + type = number + default = 300 # 5 minutes (Helm CLI default) +} + +variable "cleanup_on_fail" { + description = "Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail')." + type = bool + default = false +} + +variable "dependency_update" { + description = "Run 'helm dependency update' before installing the chart (useful if chart_name is a local path to an unpacked chart with dependencies)." + type = bool + default = false +} + +variable "disable_crd_hooks" { + description = "Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook')." + type = bool + default = false +} + +variable "disable_openapi_validation" { + description = "If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation')." + type = bool + default = false +} + +variable "disable_webhooks" { + description = "Prevent hooks from running ('helm install --no-hooks')." + type = bool + default = false +} + +variable "force_update" { + description = "Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution." + type = bool + default = false +} + +variable "lint" { + description = "Run the helm chart linter during the plan ('helm lint')." + type = bool + default = false +} + +variable "max_history" { + description = "Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit." + type = number + default = null # Terraform provider defaults to Helm's default (usually 10) +} + +variable "recreate_pods" { + description = "Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself." + type = bool + default = false +} + +variable "render_subchart_notes" { + description = "If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes')." + type = bool + default = false +} + +variable "reset_values" { + description = "When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values')." + type = bool + default = false +} + +variable "reuse_values" { + description = "When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset_values' is specified, this is ignored." + type = bool + default = false # Helm CLI default is false +} + +variable "skip_crds" { + description = "If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present." + type = bool + default = false +} + +# --- Verification & Credentials --- +variable "keyring" { + description = "Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true." + type = string + default = null # Defaults to Helm's default keyring location +} + +variable "pass_credentials" { + description = "Pass credentials to all domains ('helm install --pass-credentials'). Use with caution." + type = bool + default = false +} + +variable "verify" { + description = "Verify the package before installing it ('helm install --verify')." + type = bool + default = false +} + +# --- Advanced Rendering --- +variable "postrender" { + description = "Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary_path' attribute." + type = object({ + binary_path = string # Path to the post-renderer executable + }) + default = null # Disabled by default +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf new file mode 100644 index 0000000000..09d912e2c9 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf @@ -0,0 +1,24 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + helm = { + source = "hashicorp/helm" + version = "~> 2.17" + } + } + + required_version = ">= 1.3" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md new file mode 100644 index 0000000000..46bfe51a32 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md @@ -0,0 +1,40 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [kubernetes](#requirement\_kubernetes) | ~> 2.23 | + +## Providers + +| Name | Version | +|------|---------| +| [kubernetes](#provider\_kubernetes) | ~> 2.23 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [kubernetes_manifest.apply_manifests](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [content](#input\_content) | The YAML body to apply to gke cluster. | `string` | `null` | no | +| [field\_manager](#input\_field\_manager) | (Optional) Configure field manager options. The `name` is the name of the field manager. The `force_conflicts` flag allows overriding conflicts. | 
object({
    name            = optional(string, null)
    force_conflicts = optional(bool, false)
  })
| `null` | no | +| [resource\_timeouts](#input\_resource\_timeouts) | (Optional) Configure custom timeouts for the create, update, and delete operations of the resource. These timeouts also govern the duration for any 'wait' conditions to be met. | 
object({
    create = optional(string, null)
    update = optional(string, null)
    delete = optional(string, null)
  })
| 
{
  "create": "15m",
  "delete": "5m",
  "update": "10m"
}
| no | +| [source\_path](#input\_source\_path) | The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file. | `string` | `""` | no | +| [template\_vars](#input\_template\_vars) | The values to populate template file(s) with. | `any` | `null` | no | +| [wait\_for\_fields](#input\_wait\_for\_fields) | (Optional) A map of attribute paths and desired patterns to be matched. After each apply the provider will wait for all attributes listed here to reach a value that matches the desired pattern. | `map(string)` | `{}` | no | +| [wait\_for\_rollout](#input\_wait\_for\_rollout) | Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details. | `bool` | `true` | no | + +## Outputs + +No outputs. + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf new file mode 100644 index 0000000000..f97f26038d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf @@ -0,0 +1,104 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + yaml_separator = "\n---" + + # --- 1. Determine the primary source of YAML content --- + # Prioritize 'content' variable if provided + primary_content_body = var.content != "" ? var.content : null + + # --- 2. Handle 'source_path' based on its type (File vs. Directory) --- + + # Check if source_path is a directory (indicated by trailing slash) + is_directory = endswith(var.source_path, "/") + directory_absolute_path = local.is_directory ? abspath(var.source_path) : null + + # Check if source_path is a single yaml or tftpl file (only if not a directory) + is_single_file = !local.is_directory && ( + length(regexall("\\.yaml$", lower(var.source_path))) > 0 || + length(regexall("\\.tftpl$", lower(var.source_path))) > 0 + ) + single_file_raw_content = local.is_single_file ? ( + length(regexall("\\.tftpl$", lower(var.source_path))) > 0 ? + templatefile(abspath(var.source_path), var.template_vars) : + file(abspath(var.source_path)) + ) : null + + # Docs from primary_content_body + docs_from_primary_source = [ + for doc in split(local.yaml_separator, coalesce(local.primary_content_body, local.single_file_raw_content, "")) : trimspace(doc) + if length(trimspace(doc)) > 0 + ] + + # Docs from .yaml files in a directory + directory_yaml_files = local.is_directory ? fileset(local.directory_absolute_path, "*.yaml") : [] + docs_from_directory_yamls = flatten([ + for file_name in local.directory_yaml_files : + [ + for doc in split(local.yaml_separator, file(format("%s/%s", local.directory_absolute_path, file_name))) : trimspace(doc) + if length(trimspace(doc)) > 0 + ] + ]) + + # Docs from .tftpl files in a directory + directory_template_files = local.is_directory ? fileset(local.directory_absolute_path, "*.tftpl") : [] + docs_from_directory_templates = flatten([ + for file_name in local.directory_template_files : + [ + for doc in split(local.yaml_separator, templatefile(format("%s/%s", local.directory_absolute_path, file_name), var.template_vars)) : trimspace(doc) + if length(trimspace(doc)) > 0 + ] + ]) + + all_parsed_docs = concat( + local.docs_from_primary_source, + local.docs_from_directory_yamls, + local.docs_from_directory_templates + ) + + # --- 5. Create the final map for `for_each` (keys must be unique strings) --- + docs_map = tomap({ + for index, doc in local.all_parsed_docs : index => doc + if length(trimspace(doc)) > 0 + }) +} + +# Apply all manifest files dynamically +resource "kubernetes_manifest" "apply_manifests" { + for_each = local.docs_map + manifest = yamldecode(each.value) + timeouts { + create = var.resource_timeouts.create + update = var.resource_timeouts.update + delete = var.resource_timeouts.delete + } + + dynamic "wait" { + for_each = var.wait_for_rollout ? [1] : [] + content { + rollout = var.wait_for_rollout + fields = var.wait_for_fields + } + } + + # Configure the 'field_manager' block dynamically + dynamic "field_manager" { + for_each = var.field_manager != null ? [var.field_manager] : [] + content { + name = field_manager.value.name + force_conflicts = field_manager.value.force_conflicts + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml new file mode 100644 index 0000000000..e18197e2b7 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf new file mode 100644 index 0000000000..0b846189ea --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf @@ -0,0 +1,69 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Description: Input variables for the generic Helm release module. + +variable "content" { + description = "The YAML body to apply to gke cluster." + type = string + default = null +} + +variable "source_path" { + description = "The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file." + type = string + default = "" +} + +variable "template_vars" { + description = "The values to populate template file(s) with." + type = any + default = null +} + +variable "wait_for_rollout" { + description = "Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details." + type = bool + default = true +} + + +variable "wait_for_fields" { + description = "(Optional) A map of attribute paths and desired patterns to be matched. After each apply the provider will wait for all attributes listed here to reach a value that matches the desired pattern." + type = map(string) + default = {} +} + +variable "resource_timeouts" { + description = "(Optional) Configure custom timeouts for the create, update, and delete operations of the resource. These timeouts also govern the duration for any 'wait' conditions to be met." + type = object({ + create = optional(string, null) + update = optional(string, null) + delete = optional(string, null) + }) + default = { + create = "15m" # Default create timeout, also covers waiting for initial conditions + update = "10m" # Default update timeout, also covers waiting for update conditions + delete = "5m" # Default delete timeout + } +} + +variable "field_manager" { + description = "(Optional) Configure field manager options. The `name` is the name of the field manager. The `force_conflicts` flag allows overriding conflicts." + type = object({ + name = optional(string, null) + force_conflicts = optional(bool, false) + }) + default = null +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf new file mode 100644 index 0000000000..61786b06de --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf @@ -0,0 +1,24 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + # Defines the providers that this module depends on and their versions. + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.23" + } + } + required_version = ">= 1.3" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/main.tf new file mode 100644 index 0000000000..8db4870452 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/main.tf @@ -0,0 +1,183 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_id_parts = split("/", var.cluster_id) + cluster_name = local.cluster_id_parts[5] + cluster_location = local.cluster_id_parts[3] + project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] + + install_gpu_operator = try(var.gpu_operator.install, false) + install_nvidia_dra_driver = try(var.nvidia_dra_driver.install, false) +} + +data "google_container_cluster" "gke_cluster" { + project = local.project_id + name = local.cluster_name + location = local.cluster_location +} + +data "google_client_config" "default" {} + +module "install_kueue" { + source = "./helm_install" + depends_on = [var.gke_cluster_exists] + + release_name = "kueue" + + chart_name = "oci://registry.k8s.io/kueue/charts/kueue" + chart_version = var.kueue.version # Specify your desired Kueue version + + create_namespace = true # Helm can also create the namespace + wait = true + timeout = 600 # seconds +} + +module "install_jobset" { + source = "./helm_install" + depends_on = [var.gke_cluster_exists, module.install_kueue] + release_name = "jobset-controller" # The release name for your JobSet installation + chart_name = "oci://registry.k8s.io/jobset/charts/jobset" # The Helm repository URL for nvidia charts + chart_version = var.jobset.version + create_namespace = true + namespace = "jobset-system" +} + +module "install_nvidia_dra_driver" { + count = local.install_nvidia_dra_driver ? 1 : 0 + depends_on = [var.gke_cluster_exists] + source = "./helm_install" + + release_name = "nvidia-dra-driver-gpu" # The release name + chart_repository = "https://helm.ngc.nvidia.com/nvidia" # The Helm repository URL for nvidia charts + chart_name = "nvidia-dra-driver-gpu" # The chart name + chart_version = var.nvidia_dra_driver.version # The chart version + namespace = "nvidia-dra-driver-gpu" # The target namespace + create_namespace = true # Equivalent to --create-namespace + + # Use the 'values' argument to pass the YAML content + # This corresponds to the -f <(cat < +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.2 | +| [google](#requirement\_google) | >= 3.83 | +| [google-beta](#requirement\_google-beta) | >= 3.83 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [google-beta](#provider\_google-beta) | >= 3.83 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_global_address.private_ip_alloc](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_global_address) | resource | +| [google_service_networking_connection.private_vpc_connection](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_networking_connection) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [address](#input\_address) | The IP address or beginning of the address range allocated for the private service access. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to supporting resources. Key-value pairs. | `map(string)` | n/a | yes | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to configure private service Access.:
`projects//global/networks/`" | `string` | n/a | yes | +| [prefix\_length](#input\_prefix\_length) | The prefix length of the IP range allocated for the private service access. | `number` | `16` | no | +| [project\_id](#input\_project\_id) | ID of project in which Private Service Access will be created. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [cidr\_range](#output\_cidr\_range) | CIDR range of the created google\_compute\_global\_address | +| [connect\_mode](#output\_connect\_mode) | Services that use Private Service Access typically specify connect\_mode
"PRIVATE\_SERVICE\_ACCESS". This output value sets connect\_mode and additionally
blocks terraform actions until the VPC connection has been created. | +| [private\_vpc\_connection\_peering](#output\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection that was created by the service provider. | +| [reserved\_ip\_range](#output\_reserved\_ip\_range) | Named IP range to be used by services connected with Private Service Access. | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/main.tf new file mode 100644 index 0000000000..4bb54821c3 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/main.tf @@ -0,0 +1,42 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "private-service-access", ghpc_role = "network" }) +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_compute_global_address" "private_ip_alloc" { + provider = google-beta + name = "global-psconnect-ip-${random_id.resource_name_suffix.hex}" + project = var.project_id + purpose = "VPC_PEERING" + address_type = "INTERNAL" + network = var.network_id + prefix_length = var.prefix_length + labels = local.labels + address = var.address +} + +resource "google_service_networking_connection" "private_vpc_connection" { + network = var.network_id + service = "servicenetworking.googleapis.com" + reserved_peering_ranges = [google_compute_global_address.private_ip_alloc.name] +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/metadata.yaml new file mode 100644 index 0000000000..93e8b3970e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - servicenetworking.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/outputs.tf new file mode 100644 index 0000000000..296f2e9140 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/outputs.tf @@ -0,0 +1,43 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "private_vpc_connection_peering" { + description = "The name of the VPC Network peering connection that was created by the service provider." + sensitive = true + value = google_service_networking_connection.private_vpc_connection.peering +} + +output "connect_mode" { + description = <<-EOT + Services that use Private Service Access typically specify connect_mode + "PRIVATE_SERVICE_ACCESS". This output value sets connect_mode and additionally + blocks terraform actions until the VPC connection has been created. + EOT + value = "PRIVATE_SERVICE_ACCESS" + depends_on = [ + google_service_networking_connection.private_vpc_connection, + ] +} + +output "reserved_ip_range" { + description = "Named IP range to be used by services connected with Private Service Access." + value = google_compute_global_address.private_ip_alloc.name +} + +output "cidr_range" { + description = "CIDR range of the created google_compute_global_address" + value = "${google_compute_global_address.private_ip_alloc.address}/${google_compute_global_address.private_ip_alloc.prefix_length}" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/variables.tf new file mode 100644 index 0000000000..18b73ac2d9 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/variables.tf @@ -0,0 +1,47 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "address" { + description = "The IP address or beginning of the address range allocated for the private service access." + type = string + default = null +} + +variable "network_id" { + description = <<-EOT + The ID of the GCE VPC network to configure private service Access.: + `projects//global/networks/`" + EOT + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "labels" { + description = "Labels to add to supporting resources. Key-value pairs." + type = map(string) +} + +variable "prefix_length" { + description = "The prefix length of the IP range allocated for the private service access." + type = number + default = 16 +} + +variable "project_id" { + description = "ID of project in which Private Service Access will be created." + type = string +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/versions.tf new file mode 100644 index 0000000000..c5a889017f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/versions.tf @@ -0,0 +1,41 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 3.83" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.57.0" + } + + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.57.0" + } + + required_version = ">= 1.2" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/new-project/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/project/new-project/README.md new file mode 100644 index 0000000000..5e5cabe9d5 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/project/new-project/README.md @@ -0,0 +1,128 @@ +## Description + +This module allows you to create opinionated Google Cloud Platform projects. It +creates projects and configures aspects like Shared VPC connectivity, IAM +access, Service Accounts, and API enablement to follow best practices. + +This module is meant for use with Terraform 0.13. + +**Note:** This module has been removed from the Cluster Toolkit. The upstream module (`terraform-google-project-factory`) is now the recommended way to create and manage GCP projects. + +### Example + +```yaml +- id: project + source: github.com/terraform-google-modules/terraform-google-project-factory?rev=v17.0.0&depth=1 +``` + +This creates a new project with pre-defined project ID, a designated folder and +organization and associated billing account which will be used to pay for +services consumed. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [project\_factory](#module\_project\_factory) | terraform-google-modules/project-factory/google | ~> 11.3 | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [activate\_api\_identities](#input\_activate\_api\_identities) | The list of service identities (Google Managed service account for the API) to force-create for the project (e.g. in order to grant additional roles).
APIs in this list will automatically be appended to `activate_apis`.
Not including the API in this list will follow the default behaviour for identity creation (which is usually when the first resource using the API is created).
Any roles (e.g. service agent role) must be explicitly listed. See https://cloud.google.com/iam/docs/understanding-roles#service-agent-roles-roles for a list of related roles. | 
list(object({
    api   = string
    roles = list(string)
  }))
| `[]` | no | +| [activate\_apis](#input\_activate\_apis) | The list of apis to activate within the project | `list(string)` | 
[
  "compute.googleapis.com",
  "serviceusage.googleapis.com",
  "storage.googleapis.com"
]
| no | +| [auto\_create\_network](#input\_auto\_create\_network) | Create the default network | `bool` | `false` | no | +| [billing\_account](#input\_billing\_account) | The ID of the billing account to associate this project with | `string` | n/a | yes | +| [bucket\_force\_destroy](#input\_bucket\_force\_destroy) | Force the deletion of all objects within the GCS bucket when deleting the bucket (optional) | `bool` | `false` | no | +| [bucket\_labels](#input\_bucket\_labels) | A map of key/value label pairs to assign to the bucket (optional) | `map(string)` | `{}` | no | +| [bucket\_location](#input\_bucket\_location) | The location for a GCS bucket to create (optional) | `string` | `"US"` | no | +| [bucket\_name](#input\_bucket\_name) | A name for a GCS bucket to create (in the bucket\_project project), useful for Terraform state (optional) | `string` | `""` | no | +| [bucket\_project](#input\_bucket\_project) | A project to create a GCS bucket (bucket\_name) in, useful for Terraform state (optional) | `string` | `""` | no | +| [bucket\_ula](#input\_bucket\_ula) | Enable Uniform Bucket Level Access | `bool` | `true` | no | +| [bucket\_versioning](#input\_bucket\_versioning) | Enable versioning for a GCS bucket to create (optional) | `bool` | `false` | no | +| [budget\_alert\_pubsub\_topic](#input\_budget\_alert\_pubsub\_topic) | The name of the Cloud Pub/Sub topic where budget related messages will be published, in the form of `projects/{project_id}/topics/{topic_id}` | `string` | `null` | no | +| [budget\_alert\_spent\_percents](#input\_budget\_alert\_spent\_percents) | A list of percentages of the budget to alert on when threshold is exceeded | `list(number)` | 
[
  0.5,
  0.7,
  1
]
| no | +| [budget\_amount](#input\_budget\_amount) | The amount to use for a budget alert | `number` | `null` | no | +| [budget\_display\_name](#input\_budget\_display\_name) | The display name of the budget. If not set defaults to `Budget For ` | `string` | `null` | no | +| [budget\_monitoring\_notification\_channels](#input\_budget\_monitoring\_notification\_channels) | A list of monitoring notification channels in the form `[projects/{project_id}/notificationChannels/{channel_id}]`. A maximum of 5 channels are allowed. | `list(string)` | `[]` | no | +| [consumer\_quotas](#input\_consumer\_quotas) | The quotas configuration you want to override for the project. | 
list(object({
    service = string,
    metric  = string,
    limit   = string,
    value   = string,
  }))
| `[]` | no | +| [create\_project\_sa](#input\_create\_project\_sa) | Whether the default service account for the project shall be created | `bool` | `true` | no | +| [default\_network\_tier](#input\_default\_network\_tier) | Default Network Service Tier for resources created in this project. If unset, the value will not be modified. See https://cloud.google.com/network-tiers/docs/using-network-service-tiers and https://cloud.google.com/network-tiers. | `string` | `""` | no | +| [default\_service\_account](#input\_default\_service\_account) | Project default service account setting: can be one of `delete`, `deprivilege`, `disable`, or `keep`. | `string` | `"keep"` | no | +| [disable\_dependent\_services](#input\_disable\_dependent\_services) | Whether services that are enabled and which depend on this service should also be disabled when this service is destroyed. | `bool` | `true` | no | +| [disable\_services\_on\_destroy](#input\_disable\_services\_on\_destroy) | Whether project services will be disabled when the resources are destroyed | `bool` | `true` | no | +| [domain](#input\_domain) | The domain name (optional). | `string` | `""` | no | +| [enable\_shared\_vpc\_host\_project](#input\_enable\_shared\_vpc\_host\_project) | If this project is a shared VPC host project. If true, you must *not* set svpc\_host\_project\_id variable. Default is false. | `bool` | `false` | no | +| [folder\_id](#input\_folder\_id) | The ID of a folder to host this project | `string` | `""` | no | +| [grant\_services\_network\_role](#input\_grant\_services\_network\_role) | Whether or not to grant service agents the network roles on the host project | `bool` | `true` | no | +| [grant\_services\_security\_admin\_role](#input\_grant\_services\_security\_admin\_role) | Whether or not to grant Kubernetes Engine Service Agent the Security Admin role on the host project so it can manage firewall rules | `bool` | `false` | no | +| [group\_name](#input\_group\_name) | A group to control the project by being assigned group\_role (defaults to project editor) | `string` | `""` | no | +| [group\_role](#input\_group\_role) | The role to give the controlling group (group\_name) over the project (defaults to project editor) | `string` | `"roles/editor"` | no | +| [labels](#input\_labels) | Map of labels for project | `map(string)` | `{}` | no | +| [lien](#input\_lien) | Add a lien on the project to prevent accidental deletion | `bool` | `false` | no | +| [name](#input\_name) | The name for the project | `string` | `null` | no | +| [org\_id](#input\_org\_id) | The organization ID. | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | The ID to give the project. If not provided, the `name` will be used. | `string` | `""` | no | +| [project\_sa\_name](#input\_project\_sa\_name) | Default service account name for the project. | `string` | `"project-service-account"` | no | +| [random\_project\_id](#input\_random\_project\_id) | Adds a suffix of 4 random characters to the `project_id` | `bool` | `false` | no | +| [sa\_role](#input\_sa\_role) | A role to give the default Service Account for the project (defaults to none) | `string` | `""` | no | +| [shared\_vpc\_subnets](#input\_shared\_vpc\_subnets) | List of subnets fully qualified subnet IDs (ie. projects/$project\_id/regions/$region/subnetworks/$subnet\_id) | `list(string)` | `[]` | no | +| [svpc\_host\_project\_id](#input\_svpc\_host\_project\_id) | The ID of the host project which hosts the shared VPC | `string` | `""` | no | +| [usage\_bucket\_name](#input\_usage\_bucket\_name) | Name of a GCS bucket to store GCE usage reports in (optional) | `string` | `""` | no | +| [usage\_bucket\_prefix](#input\_usage\_bucket\_prefix) | Prefix in the GCS bucket to store GCE usage reports in (optional) | `string` | `""` | no | +| [vpc\_service\_control\_attach\_enabled](#input\_vpc\_service\_control\_attach\_enabled) | Whether the project will be attached to a VPC Service Control Perimeter | `bool` | `false` | no | +| [vpc\_service\_control\_perimeter\_name](#input\_vpc\_service\_control\_perimeter\_name) | The name of a VPC Service Control Perimeter to add the created project to | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [api\_s\_account](#output\_api\_s\_account) | API service account email | +| [api\_s\_account\_fmt](#output\_api\_s\_account\_fmt) | API service account email formatted for terraform use | +| [budget\_name](#output\_budget\_name) | The name of the budget if created | +| [domain](#output\_domain) | The organization's domain | +| [enabled\_api\_identities](#output\_enabled\_api\_identities) | Enabled API identities in the project | +| [enabled\_apis](#output\_enabled\_apis) | Enabled APIs in the project | +| [group\_email](#output\_group\_email) | The email of the G Suite group with group\_name | +| [project\_bucket\_self\_link](#output\_project\_bucket\_self\_link) | Project's bucket selfLink | +| [project\_bucket\_url](#output\_project\_bucket\_url) | Project's bucket url | +| [project\_id](#output\_project\_id) | ID of the project that was created | +| [project\_name](#output\_project\_name) | Name of the project that was created | +| [project\_number](#output\_project\_number) | Number of the project that was created | +| [service\_account\_display\_name](#output\_service\_account\_display\_name) | The display name of the default service account | +| [service\_account\_email](#output\_service\_account\_email) | The email of the default service account | +| [service\_account\_id](#output\_service\_account\_id) | The id of the default service account | +| [service\_account\_name](#output\_service\_account\_name) | The fully-qualified name of the default service account | +| [service\_account\_unique\_id](#output\_service\_account\_unique\_id) | The unique id of the default service account | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/README.md new file mode 100644 index 0000000000..0f5c10c7e4 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/README.md @@ -0,0 +1,111 @@ +## Description + +Allows creation of service accounts for a Google Cloud Platform project. + +### Example + +```yaml +- id: service_acct + source: community/modules/project/service-account + settings: + project_id: $(vars.project_id) + name: instance_acct + project_roles: + - logging.logWriter + - monitoring.metricWriter + - storage.objectViewer +``` + +This creates a service account in GCP project "project_id" with the name +"instance_acct". It will have the 3 roles listed for all resources within the +project. + +### Usage with startup-script module + +When this module is used in conjunction with the [startup-script] module, the +service account must be granted (at least) read access to the bucket. This can +be achieved by granting project-wide access as shown above or by specifying the +service account as a bucket viewer in the startup-script module: + +```yaml +- id: service_acct + source: community/modules/project/service-account + settings: + project_id: $(vars.project_id) + name: instance_acct + project_roles: + - logging.logWriter + - monitoring.metricWriter +- id: script + source: modules/scripts/startup-script + settings: + bucket_viewers: + - $(service_acct.service_account_iam_email) +``` + +[startup-script]: ../../../../modules/scripts/startup-script/README.md + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [service\_account](#module\_service\_account) | terraform-google-modules/service-accounts/google | ~> 4.2 | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [billing\_account\_id](#input\_billing\_account\_id) | If assigning billing role, specify a billing account (default is to assign at the organizational level). | `string` | `""` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment (will be prepended to service account name) | `string` | n/a | yes | +| [description](#input\_description) | Description of the created service account. | `string` | `"Service Account"` | no | +| [descriptions](#input\_descriptions) | Deprecated; create single service accounts using var.description. | `list(string)` | `null` | no | +| [display\_name](#input\_display\_name) | Display name of the created service account. | `string` | `"Service Account"` | no | +| [generate\_keys](#input\_generate\_keys) | Generate keys for service account. | `bool` | `false` | no | +| [grant\_billing\_role](#input\_grant\_billing\_role) | Grant billing user role. | `bool` | `false` | no | +| [grant\_xpn\_roles](#input\_grant\_xpn\_roles) | Grant roles for shared VPC management. | `bool` | `true` | no | +| [name](#input\_name) | Name of the service account to create. | `string` | n/a | yes | +| [names](#input\_names) | Deprecated; create single service accounts using var.name. | `list(string)` | `null` | no | +| [org\_id](#input\_org\_id) | Id of the organization for org-level roles. | `string` | `""` | no | +| [prefix](#input\_prefix) | Deprecated; prefix now set using var.deployment\_name | `string` | `null` | no | +| [project\_id](#input\_project\_id) | ID of the project | `string` | n/a | yes | +| [project\_roles](#input\_project\_roles) | List of roles to grant to service account (e.g. "storage.objectViewer" or "compute.instanceAdmin.v1" | `list(string)` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [key](#output\_key) | Service account key (if creation was requested) | +| [service\_account\_email](#output\_service\_account\_email) | Service account e-mail address | +| [service\_account\_iam\_email](#output\_service\_account\_iam\_email) | Service account IAM binding format (serviceAccount:name@example.com) | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/main.tf new file mode 100644 index 0000000000..e8a69be642 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/main.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + display_name = "${var.display_name} (${var.deployment_name})" + description = "${var.description} (${var.deployment_name})" +} + +module "service_account" { + source = "terraform-google-modules/service-accounts/google" + version = "~> 4.2" + + billing_account_id = var.billing_account_id + description = local.description + display_name = local.display_name + generate_keys = var.generate_keys + grant_billing_role = var.grant_billing_role + grant_xpn_roles = var.grant_xpn_roles + names = [var.name] + org_id = var.org_id + prefix = var.deployment_name + project_id = var.project_id + project_roles = [for role in var.project_roles : "${var.project_id}=>roles/${role}"] +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/metadata.yaml new file mode 100644 index 0000000000..c4dcdffdf4 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - iam.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/outputs.tf new file mode 100644 index 0000000000..f9c9be05c8 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/outputs.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "key" { + description = "Service account key (if creation was requested)" + value = module.service_account.key +} + +output "service_account_email" { + description = "Service account e-mail address" + value = module.service_account.email + depends_on = [ + module.service_account, + ] +} + +output "service_account_iam_email" { + description = "Service account IAM binding format (serviceAccount:name@example.com)" + value = module.service_account.iam_email + depends_on = [ + module.service_account, + ] +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/variables.tf new file mode 100644 index 0000000000..53267f47e7 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/variables.tf @@ -0,0 +1,113 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "billing_account_id" { + description = "If assigning billing role, specify a billing account (default is to assign at the organizational level)." + type = string + default = "" +} + +variable "deployment_name" { + description = "Name of the deployment (will be prepended to service account name)" + type = string +} + +variable "description" { + description = "Description of the created service account." + type = string + default = "Service Account" +} + +# tflint-ignore: terraform_unused_declarations +variable "descriptions" { + description = "Deprecated; create single service accounts using var.description." + type = list(string) + default = null + + validation { + condition = var.descriptions == null + error_message = "var.descriptions has been deprecated in favor of creating single accounts with var.description" + } +} + +variable "display_name" { + description = "Display name of the created service account." + type = string + default = "Service Account" +} + +variable "generate_keys" { + description = "Generate keys for service account." + type = bool + default = false +} + +variable "grant_billing_role" { + description = "Grant billing user role." + type = bool + default = false +} + +variable "grant_xpn_roles" { + description = "Grant roles for shared VPC management." + type = bool + default = true +} + +variable "name" { + description = "Name of the service account to create." + type = string +} + +# tflint-ignore: terraform_unused_declarations +variable "names" { + description = "Deprecated; create single service accounts using var.name." + type = list(string) + default = null + + validation { + condition = var.names == null + error_message = "var.names has been deprecated in favor of creating single accounts with var.name" + } +} + +variable "org_id" { + description = "Id of the organization for org-level roles." + type = string + default = "" +} + +# tflint-ignore: terraform_unused_declarations +variable "prefix" { + description = "Deprecated; prefix now set using var.deployment_name" + type = string + default = null + + validation { + condition = var.prefix == null + error_message = "var.prefix has been deprecated in favor of setting prefix with var.deployment_name" + } +} + +variable "project_id" { + description = "ID of the project" + type = string +} + +variable "project_roles" { + description = "List of roles to grant to service account (e.g. \"storage.objectViewer\" or \"compute.instanceAdmin.v1\"" + type = list(string) +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/versions.tf new file mode 100644 index 0000000000..38e6e71945 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/versions.tf @@ -0,0 +1,22 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + } + + required_version = ">= 0.14.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/README.md new file mode 100644 index 0000000000..266eac26ec --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/README.md @@ -0,0 +1,70 @@ +## Description + +Allows management of multiple API services for a Google Cloud Platform project. + +### Example + +```yaml +- id: services-api + source: community/modules/project/service-enablement + settings: + gcp_service_list: [ + "file.googleapis.com", + "compute.googleapis.com" + ] +``` + +This allows the project to enable both the filestore API as well as the compute API. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_project_service.gcp_services](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_service) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [disable\_on\_destroy](#input\_disable\_on\_destroy) | Disable services on destroy if they were enabled (or already enabled) during apply (default: false) | `bool` | `false` | no | +| [gcp\_service\_list](#input\_gcp\_service\_list) | list of APIs to be enabled for the project | `list(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | ID of the project | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/main.tf new file mode 100644 index 0000000000..965e93c549 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/main.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +resource "google_project_service" "gcp_services" { + count = length(var.gcp_service_list) + project = var.project_id + service = var.gcp_service_list[count.index] + timeouts { + create = "30m" + update = "40m" + } + + disable_dependent_services = true + disable_on_destroy = var.disable_on_destroy +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/metadata.yaml new file mode 100644 index 0000000000..c594c8f819 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - serviceusage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/variables.tf new file mode 100644 index 0000000000..08f13999fe --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/variables.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "ID of the project" + type = string +} + +variable "gcp_service_list" { + description = "list of APIs to be enabled for the project" + type = list(string) +} + +variable "disable_on_destroy" { + description = "Disable services on destroy if they were enabled (or already enabled) during apply (default: false)" + type = bool + default = false +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/versions.tf new file mode 100644 index 0000000000..67eb446cde --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:service-enablement/v1.57.0" + } + + required_version = ">= 0.14.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/README.md new file mode 100644 index 0000000000..052e6aee23 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/README.md @@ -0,0 +1,87 @@ +# Description + +This module creates a Bigquery Pub/Sub Subscription. + +Primarily used for FSI - MonteCarlo Tutorial: +**[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +## Example + +The following example creates a Bigquery subscription using a Bigquery table and +Pub/Sub topic. + +```yaml + - id: bq_subscription + source: community/modules/pubsub/bigquery-sub + use: [bq-table, pubsub_topic] +``` + +Also see usages in this +[example blueprint](../../../examples/fsi-montecarlo-on-batch.yaml). + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_project_iam_member.editor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | +| [google_project_iam_member.viewer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | +| [google_pubsub_subscription.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [dataset\_id](#input\_dataset\_id) | Name of the dataset that was created. Can be provided by the bigquery-table module | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [subscription\_id](#input\_subscription\_id) | The name of the pubsub subscription to be created | `string` | `null` | no | +| [table\_id](#input\_table\_id) | ID of created BQ table. Can be provided by the bigquery-table module | `string` | n/a | yes | +| [topic\_id](#input\_topic\_id) | The name of the pubsub topic to subscribe to. Can be provided by the pubsub/topic module | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [subscription\_id](#output\_subscription\_id) | Name of the subscription that was created. | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf new file mode 100644 index 0000000000..8edbc6b24e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf @@ -0,0 +1,57 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "bigquery-sub", ghpc_role = "pubsub" }) +} + +locals { + subscription_id = var.subscription_id != null ? var.subscription_id : "${var.deployment_name}_subscription_${random_id.resource_name_suffix.hex}" +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} +data "google_project" "project" { + project_id = var.project_id +} + +resource "google_project_iam_member" "viewer" { + project = data.google_project.project.project_id + role = "roles/bigquery.metadataViewer" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" +} + +resource "google_project_iam_member" "editor" { + project = data.google_project.project.project_id + role = "roles/bigquery.dataEditor" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" +} + +resource "google_pubsub_subscription" "example" { + depends_on = [google_project_iam_member.editor, google_project_iam_member.viewer] + name = local.subscription_id + topic = var.topic_id + project = var.project_id + labels = local.labels + bigquery_config { + table = "${var.project_id}.${var.dataset_id}.${var.table_id}" + use_topic_schema = true + write_metadata = true + } + +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml new file mode 100644 index 0000000000..9aedef48dc --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - pubsub.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf new file mode 100644 index 0000000000..fc81859503 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "subscription_id" { + description = "Name of the subscription that was created." + value = google_pubsub_subscription.example.name +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf new file mode 100644 index 0000000000..ee4dbbed8e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "topic_id" { + description = "The name of the pubsub topic to subscribe to. Can be provided by the pubsub/topic module" + type = string +} + +variable "subscription_id" { + description = "The name of the pubsub subscription to be created" + type = string + default = null +} + +variable "dataset_id" { + description = "Name of the dataset that was created. Can be provided by the bigquery-table module" + type = string +} + +variable "table_id" { + description = "ID of created BQ table. Can be provided by the bigquery-table module" + type = string +} + +variable "labels" { + description = "Labels to add to the instances. Key-value pairs." + type = map(string) +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf new file mode 100644 index 0000000000..54f9ccf8bb --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.57.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.57.0" + } + required_version = ">= 1.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/README.md new file mode 100644 index 0000000000..177f799dc6 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/README.md @@ -0,0 +1,82 @@ +## Description + +Creates a Pub/Sub topic + +Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. + +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +### Example + +The following example creates a Pub/Sub topic. + +```yaml + - id: pubsub_topic + source: community/modules/pubsub/topic +``` + +Also see usages in this +[example blueprint](../../../examples/fsi-montecarlo-on-batch.yaml). + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_pubsub_schema.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_schema) | resource | +| [google_pubsub_topic.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [schema\_id](#input\_schema\_id) | The name of the pubsub schema to be created | `string` | `null` | no | +| [schema\_json](#input\_schema\_json) | The JSON definition of the pubsub topic schema | `string` | `"{ \n \"name\" : \"Avro\", \n \"type\" : \"record\", \n \"fields\" : \n [\n {\"name\" : \"ticker\", \"type\" : \"string\"},\n {\"name\" : \"epoch_time\", \"type\" : \"int\"},\n {\"name\" : \"iteration\", \"type\" : \"int\"},\n {\"name\" : \"start_date\", \"type\" : \"string\"},\n {\"name\" : \"end_date\", \"type\" : \"string\"},\n {\n \"name\":\"simulation_results\",\n \"type\":{\n \"type\": \"array\", \n \"items\":{\n \"name\":\"Child\",\n \"type\":\"record\",\n \"fields\":[\n {\"name\":\"price\", \"type\":\"double\"}\n ]\n }\n }\n }\n ]\n }\n"` | no | +| [topic\_id](#input\_topic\_id) | The name of the pubsub topic to be created | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [topic\_id](#output\_topic\_id) | Name of the topic that was created. | +| [topic\_schema](#output\_topic\_schema) | Name of the topic schema that was created. | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/main.tf new file mode 100644 index 0000000000..4ba68fb5d0 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/main.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "topic", ghpc_role = "pubsub" }) +} + +locals { + topic_id = var.topic_id != null ? var.topic_id : "${var.deployment_name}_topic_${random_id.resource_name_suffix.hex}" + schema_id = var.schema_id != null ? var.schema_id : "${var.deployment_name}_schema_${random_id.resource_name_suffix.hex}" +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_pubsub_topic" "example" { + name = local.topic_id + depends_on = [google_pubsub_schema.example] + project = var.project_id + labels = local.labels + schema_settings { + schema = "projects/${var.project_id}/schemas/${local.schema_id}" + encoding = "BINARY" + } +} + +resource "google_pubsub_schema" "example" { + name = local.schema_id + project = var.project_id + type = "AVRO" + + definition = var.schema_json +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/metadata.yaml new file mode 100644 index 0000000000..9aedef48dc --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - pubsub.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/outputs.tf new file mode 100644 index 0000000000..3ea9d951b2 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/outputs.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "topic_id" { + description = "Name of the topic that was created." + value = google_pubsub_topic.example.name +} + + +output "topic_schema" { + description = "Name of the topic schema that was created." + value = local.schema_id +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/variables.tf new file mode 100644 index 0000000000..dca575d21d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/variables.tf @@ -0,0 +1,74 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "topic_id" { + description = "The name of the pubsub topic to be created" + type = string + default = null +} + +variable "schema_id" { + description = "The name of the pubsub schema to be created" + type = string + default = null +} + +variable "schema_json" { + description = "The JSON definition of the pubsub topic schema" + type = string + default = < **Note**: This is an experimental module. This module has only been tested in +> limited capacity with the Cluster Toolkit. The module interface may have undergo +> breaking changes in the future. + +### Example + +The following example will create a single GPU accelerated remote desktop. + +```yaml + - id: remote-desktop + source: community/modules/remote-desktop/chrome-remote-desktop + use: [network1] + settings: + install_nvidia_driver: true +``` + +### Setting up the Remote Desktop + +1. Once the remote desktop has been deployed, navigate to https://remotedesktop.google.com/headless. +1. Click through `Begin`, `Next`, & `Authorize`. +1. Copy the code snippet for `Debian Linux`. +1. SSH into the remote desktop machine. It will be listed under + [VM Instances](https://console.cloud.google.com/compute/instances) in the + Google Cloud web console. +1. Run the copied command and follow instructions to set up a PIN. +1. You should now see your machine listed on the + [Chrome Remote Desktop page](https://remotedesktop.google.com/access) under `Remote devices`. +1. Click on your machine and enter PIN if prompted. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.12.31 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [client\_startup\_script](#module\_client\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | +| [instances](#module\_instances) | ../../../../modules/compute/vm-instance | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [add\_deployment\_name\_before\_prefix](#input\_add\_deployment\_name\_before\_prefix) | If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments.
See `name_prefix` for further details on resource naming behavior. | `bool` | `false` | no | +| [auto\_delete\_boot\_disk](#input\_auto\_delete\_boot\_disk) | Controls if boot disk should be auto-deleted when instance is deleted. | `bool` | `true` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image th
at supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-v
m-with-high-bandwidth-configuration) for more details. | `string` | `"not_enabled"` | no | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. Cloud resource names will include this value. | `string` | n/a | yes | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for instances. | `number` | `200` | no | +| [disk\_type](#input\_disk\_type) | Disk type for instances. | `string` | `"pd-balanced"` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true, instances will have public IPs on the internet. | `bool` | `true` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. Requires virtual workstation accelerator if Nvidia Grid Drivers are required | 
list(object({
    type  = string,
    count = number
  }))
| 
[
  {
    "count": 1,
    "type": "nvidia-tesla-t4-vws"
  }
]
| no | +| [install\_nvidia\_driver](#input\_install\_nvidia\_driver) | Installs the nvidia driver (true/false). For details, see https://cloud.google.com/compute/docs/gpus/install-drivers-gpu | `bool` | n/a | yes | +| [instance\_count](#input\_instance\_count) | Number of instances | `number` | `1` | no | +| [instance\_image](#input\_instance\_image) | Image used to build chrome remote desktop node. The default image is
name="debian-12-bookworm-v20250610" and project="debian-cloud".
NOTE: uses fixed version of image to avoid NVIDIA driver compatibility issues.

An alternative image is from name="ubuntu-2204-jammy-v20240126" and project="ubuntu-os-cloud".

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "name": "debian-12-bookworm-v20250610",
  "project": "debian-cloud"
}
| no | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation. Must be N1 family if GPU is used. | `string` | `"n1-standard-8"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | +| [name\_prefix](#input\_name\_prefix) | An optional name for all VM and disk resources.
If not supplied, `deployment_name` will be used.
When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set,
then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". | `string` | `null` | no | +| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces. The options match that of the terraform
network\_interface block of google\_compute\_instance. For descriptions of the
subfields or more information see the documentation:
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface
**\_NOTE:\_** If `network_interfaces` are set, `network_self_link` and
`subnetwork_self_link` will be ignored, even if they are provided through
the `use` field. `bandwidth_tier` and `enable_public_ips` also do not apply
to network interfaces defined in this variable.
Subfields:
network (string, required if subnetwork is not supplied)
subnetwork (string, required if network is not supplied)
subnetwork\_project (string, optional)
network\_ip (string, optional)
nic\_type (string, optional, choose from ["GVNIC", "VIRTIO\_NET", "RDMA", "IRDMA", "MRDMA"])
stack\_type (string, optional, choose from ["IPV4\_ONLY", "IPV4\_IPV6"])
queue\_count (number, optional)
access\_config (object, optional)
ipv6\_access\_config (object, optional)
alias\_ip\_range (list(object), optional) | 
list(object({
    network            = string,
    subnetwork         = string,
    subnetwork_project = string,
    network_ip         = string,
    nic_type           = string,
    stack_type         = string,
    queue_count        = number,
    access_config = list(object({
      nat_ip                 = string,
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    ipv6_access_config = list(object({
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    alias_ip_range = list(object({
      ip_cidr_range         = string,
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the VM. | `string` | `"default"` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE` | `string` | `"TERMINATE"` | no | +| [project\_id](#input\_project\_id) | Project in which Google Cloud resources will be created | `string` | n/a | yes | +| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | +| [service\_account](#input\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account. | 
object({
    email  = string,
    scopes = set(string)
  })
| 
{
  "email": null,
  "scopes": [
    "https://www.googleapis.com/auth/cloud-platform"
  ]
}
| no | +| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | +| [startup\_script](#input\_startup\_script) | Startup script used on the instance | `string` | `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the VM. | `string` | `null` | no | +| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | +| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core | `number` | `2` | no | +| [zone](#input\_zone) | Default zone for creating resources | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [instance\_name](#output\_instance\_name) | Name of the first instance created, if any. | +| [startup\_script](#output\_startup\_script) | script to load and run all runners, as a string value. | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf new file mode 100644 index 0000000000..a5cf7c5d37 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf @@ -0,0 +1,111 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "chrome-remote-desktop", ghpc_role = "remote-desktop" }) +} + +locals { + + user_startup_script_runners = var.startup_script == null ? [] : [ + { + type = "shell" + content = var.startup_script + destination = "user_startup_script.sh" + } + ] + + configure_nvidia_driver_runners = var.install_nvidia_driver == false ? [] : [ + { + type = "ansible-local" + content = file("${path.module}/scripts/configure-grid-drivers.yml") + destination = "/usr/local/ghpc/configure-grid-drivers.yml" + } + ] + + configure_chrome_remote_desktop_runners = [ + { + type = "ansible-local" + content = file("${path.module}/scripts/configure-chrome-desktop.yml") + destination = "/usr/local/ghpc/configure-chrome-desktop.yml" + } + ] + + disable_sleep = [ + { + type = "ansible-local" + content = file("${path.module}/scripts/disable-sleep.yml") + destination = "/usr/local/ghpc/disable-sleep.yml" + } + ] +} + +module "client_startup_script" { + source = "../../../../modules/scripts/startup-script" + + deployment_name = var.deployment_name + project_id = var.project_id + region = var.region + labels = local.labels + + runners = flatten([ + local.user_startup_script_runners, + local.configure_nvidia_driver_runners, + local.configure_chrome_remote_desktop_runners, + local.disable_sleep + ]) +} + +module "instances" { + source = "../../../../modules/compute/vm-instance" + + instance_count = var.instance_count + name_prefix = var.name_prefix + add_deployment_name_before_prefix = var.add_deployment_name_before_prefix + provisioning_model = var.spot ? "SPOT" : null + + deployment_name = var.deployment_name + project_id = var.project_id + region = var.region + zone = var.zone + labels = local.labels + + machine_type = var.machine_type + service_account_email = var.service_account.email + metadata = var.metadata + startup_script = module.client_startup_script.startup_script + enable_oslogin = var.enable_oslogin + + instance_image = var.instance_image + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + auto_delete_boot_disk = var.auto_delete_boot_disk + + disable_public_ips = !var.enable_public_ips + network_self_link = var.network_self_link + subnetwork_self_link = var.subnetwork_self_link + network_interfaces = var.network_interfaces + bandwidth_tier = var.bandwidth_tier + tags = var.tags + + threads_per_core = var.threads_per_core + guest_accelerator = var.guest_accelerator + on_host_maintenance = var.on_host_maintenance + + network_storage = var.network_storage + +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf new file mode 100644 index 0000000000..bcf8ece52d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "script to load and run all runners, as a string value." + value = module.client_startup_script.startup_script +} + +output "instance_name" { + description = "Name of the first instance created, if any." + value = var.instance_count > 0 ? module.instances.name[0] : null +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml new file mode 100644 index 0000000000..391aa86433 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml @@ -0,0 +1,61 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Ensure Desktop OS and Chrome Remote Desktop is installed + hosts: localhost + become: true + module_defaults: + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + tasks: + - name: Install desktop packages + ansible.builtin.apt: + name: + - xfce4 + - xfce4-goodies + state: present + register: apt_result + retries: 10 + delay: 30 + until: apt_result is success + + - name: Download and configure CRD + ansible.builtin.get_url: + url: https://dl.google.com/linux/direct/chrome-remote-desktop_current_amd64.deb + dest: /tmp/chrome-remote-desktop_current_amd64.deb + mode: "0755" + timeout: 30 + + - name: Install CRD + ansible.builtin.apt: + deb: /tmp/chrome-remote-desktop_current_amd64.deb + environment: + DEBIAN_FRONTEND: noninteractive + register: apt_result + retries: 10 + delay: 30 + until: apt_result is success + + - name: Configure CRD to use Xfce by default + ansible.builtin.copy: + dest: /etc/chrome-remote-desktop-session + content: "exec /etc/X11/Xsession /usr/bin/xfce4-session" + mode: 0644 + + - name: Start Chrome remote desktop + ansible.builtin.command: /etc/init.d/chrome-remote-desktop start + register: result + changed_when: result.rc == 0 diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml new file mode 100644 index 0000000000..daae08176d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml @@ -0,0 +1,163 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Ensure nvidia grid drivers and other binaries are installed + hosts: localhost + become: true + vars: + dist_settings: + bullseye: + packages: + - build-essential + - gdebi-core + - mesa-utils + - gdm3 + - linux-headers-{{ ansible_kernel }} + grid_fn: NVIDIA-Linux-x86_64-510.85.02-grid.run + grid_ver: vGPU14.2 + bookworm: + packages: + - build-essential + - gdebi-core + - mesa-utils + - gdm3 + - linux-headers-{{ ansible_kernel }} + grid_fn: NVIDIA-Linux-x86_64-550.54.15-grid.run + grid_ver: vGPU17.1 + jammy: + packages: + - build-essential + - gdebi-core + - mesa-utils + - gdm3 + - gcc-12 # must match compiler used to build kernel on latest Ubuntu 22 + - pkg-config # observed to be necessary for GRID driver installation on latest Ubuntu 22 + - libglvnd-dev # observed to be necessary for GRID driver installation on latest Ubuntu 22 + - linux-headers-{{ ansible_kernel }} + grid_fn: NVIDIA-Linux-x86_64-525.125.06-grid.run + grid_ver: vGPU15.3 + tasks: + - name: Fail if using wrong OS + ansible.builtin.assert: + that: + - ansible_os_family in ["Debian", "Ubuntu"] + - ansible_distribution_release in dist_settings.keys() | list + fail_msg: "ansible_os_family: {{ ansible_os_family }} or ansible_distribution_release: {{ansible_distribution_release}} was not acceptable." + + - name: Check if GRID driver installed + ansible.builtin.command: which nvidia-smi + register: nvidiasmi_result + ignore_errors: true + changed_when: false + + - name: Install binaries for GRID drivers + ansible.builtin.apt: + name: '{{ dist_settings[ansible_distribution_release]["packages"] }}' + state: present + update_cache: true + register: apt_result + retries: 6 + delay: 10 + until: apt_result is success + + - name: Install GRID driver if not existing + when: nvidiasmi_result is failed + block: + - name: Download GPU driver + ansible.builtin.get_url: + url: https://storage.googleapis.com/nvidia-drivers-us-public/GRID/{{ dist_settings[ansible_distribution_release]["grid_ver"] }}/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} + dest: /tmp/ + mode: "0755" + timeout: 30 + + - name: Stop gdm service + ansible.builtin.systemd: + name: gdm + state: stopped + + - name: Install GPU driver + ansible.builtin.shell: | + #jinja2: trim_blocks: "True" + {% if ansible_distribution_release == "jammy" %} + CC=gcc-12 /tmp/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} --silent + {% else %} + /tmp/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} --silent + {% endif %} + register: result + changed_when: result.rc == 0 + + - name: Download VirtualGL driver + ansible.builtin.get_url: + url: https://sourceforge.net/projects/virtualgl/files/3.0.2/virtualgl_3.0.2_amd64.deb/download + dest: /tmp/virtualgl_3.0.2_amd64.deb + mode: "0755" + timeout: 30 + + - name: Install VirtualGL + ansible.builtin.command: gdebi /tmp/virtualgl_3.0.2_amd64.deb --non-interactive + register: result + changed_when: result.rc == 0 + + - name: Fix headless Nvidia issue + block: + - name: Lookup gpu info + ansible.builtin.command: nvidia-xconfig --query-gpu-info + register: gpu_info + failed_when: gpu_info.rc != 0 + changed_when: false + + - name: Extract PCI ID + ansible.builtin.shell: | + set -o pipefail + echo "{{ gpu_info.stdout }}" | grep "PCI BusID " | head -n 1 | cut -d':' -f2-99 | xargs + args: + executable: /bin/bash + register: pci_id + changed_when: false + + - name: Configure nvidia-xconfig + ansible.builtin.command: nvidia-xconfig -a --allow-empty-initial-configuration --enable-all-gpus --virtual=1920x1200 --busid={{ pci_id.stdout }} + register: result + changed_when: result.rc == 0 + + - name: Set HardDPMS to false + ansible.builtin.replace: + path: /etc/X11/xorg.conf + regexp: "Section \"Device\"" + replace: "Section \"Device\"\n Option \"HardDPMS\" \"false\"" + + - name: Configure VirtualGL for X + ansible.builtin.command: vglserver_config +glx +s +f -t + register: result + changed_when: result.rc == 0 + + - name: Configure gdm for X + block: + - name: Configure default display manager + ansible.builtin.copy: + dest: /etc/X11/default-display-manager + content: "/usr/sbin/gdm3" + mode: 0644 + + - name: Switch boot target to gui + ansible.builtin.command: systemctl set-default graphical.target + register: result + changed_when: result.rc == 0 + + - name: Start gdm service + ansible.builtin.systemd: + name: gdm + daemon_reload: true + state: started diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml new file mode 100644 index 0000000000..6767b05fb2 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml @@ -0,0 +1,39 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Mask sleep, suspend, hibernate, and hybrid-sleep targets + hosts: localhost + become: true + tasks: + + - name: Mask sleep target + ansible.builtin.systemd: + name: sleep.target + masked: true + + - name: Mask suspend target + ansible.builtin.systemd: + name: suspend.target + masked: true + + - name: Mask hibernate target + ansible.builtin.systemd: + name: hibernate.target + masked: true + + - name: Mask hybrid-sleep target + ansible.builtin.systemd: + name: hybrid-sleep.target + masked: true diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf new file mode 100644 index 0000000000..ac4c3b1869 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf @@ -0,0 +1,277 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which Google Cloud resources will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. Cloud resource names will include this value." + type = string + #default = "chrome-remote-desktop" +} + +variable "region" { + description = "Default region for creating resources" + type = string +} + +variable "zone" { + description = "Default zone for creating resources" + type = string +} + +variable "instance_count" { + description = "Number of instances" + type = number + default = 1 +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured." + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "instance_image" { + description = <<-EOD + Image used to build chrome remote desktop node. The default image is + name="debian-12-bookworm-v20250610" and project="debian-cloud". + NOTE: uses fixed version of image to avoid NVIDIA driver compatibility issues. + + An alternative image is from name="ubuntu-2204-jammy-v20240126" and project="ubuntu-os-cloud". + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + default = { + project = "debian-cloud" + name = "debian-12-bookworm-v20250610" + } +} + +variable "disk_size_gb" { + description = "Size of disk for instances." + type = number + default = 200 +} + +variable "disk_type" { + description = "Disk type for instances." + type = string + default = "pd-balanced" +} + +variable "auto_delete_boot_disk" { + description = "Controls if boot disk should be auto-deleted when instance is deleted." + type = bool + default = true +} + +variable "name_prefix" { + description = <<-EOT + An optional name for all VM and disk resources. + If not supplied, `deployment_name` will be used. + When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set, + then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". + EOT + type = string + default = null +} + +variable "add_deployment_name_before_prefix" { + description = <<-EOT + If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments. + See `name_prefix` for further details on resource naming behavior. + EOT + type = bool + default = false +} + +variable "enable_public_ips" { + description = "If set to true, instances will have public IPs on the internet." + type = bool + default = true +} + +variable "machine_type" { + description = "Machine type to use for the instance creation. Must be N1 family if GPU is used." + type = string + default = "n1-standard-8" +} + +variable "labels" { + description = "Labels to add to the instances. Key-value pairs." + type = map(string) + default = {} +} + +variable "service_account" { + description = "Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account." + type = object({ + email = string, + scopes = set(string) + }) + default = { + email = null + scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + ] + } +} + +variable "network_self_link" { + description = "The self link of the network to attach the VM." + type = string + default = "default" +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork to attach the VM." + type = string + default = null +} + +variable "network_interfaces" { + description = <<-EOT + A list of network interfaces. The options match that of the terraform + network_interface block of google_compute_instance. For descriptions of the + subfields or more information see the documentation: + https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface + **_NOTE:_** If `network_interfaces` are set, `network_self_link` and + `subnetwork_self_link` will be ignored, even if they are provided through + the `use` field. `bandwidth_tier` and `enable_public_ips` also do not apply + to network interfaces defined in this variable. + Subfields: + network (string, required if subnetwork is not supplied) + subnetwork (string, required if network is not supplied) + subnetwork_project (string, optional) + network_ip (string, optional) + nic_type (string, optional, choose from ["GVNIC", "VIRTIO_NET", "RDMA", "IRDMA", "MRDMA"]) + stack_type (string, optional, choose from ["IPV4_ONLY", "IPV4_IPV6"]) + queue_count (number, optional) + access_config (object, optional) + ipv6_access_config (object, optional) + alias_ip_range (list(object), optional) + EOT + type = list(object({ + network = string, + subnetwork = string, + subnetwork_project = string, + network_ip = string, + nic_type = string, + stack_type = string, + queue_count = number, + access_config = list(object({ + nat_ip = string, + public_ptr_domain_name = string, + network_tier = string + })), + ipv6_access_config = list(object({ + public_ptr_domain_name = string, + network_tier = string + })), + alias_ip_range = list(object({ + ip_cidr_range = string, + subnetwork_range_name = string + })) + })) + default = [] +} + +variable "metadata" { + description = "Metadata, provided as a map" + type = map(string) + default = {} +} + +variable "startup_script" { + description = "Startup script used on the instance" + type = string + default = null +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance. Requires virtual workstation accelerator if Nvidia Grid Drivers are required" + type = list(object({ + type = string, + count = number + })) + default = [{ + type = "nvidia-tesla-t4-vws" + count = 1 + }] +} + +variable "threads_per_core" { + description = "Sets the number of threads per physical core" + type = number + default = 2 +} + +variable "on_host_maintenance" { + description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE`" + type = string + default = "TERMINATE" +} + +variable "bandwidth_tier" { + description = <> --all-instances --region <> \ + --project <> --minimal-action replace +``` + +This mode can be switched to proactive (automatic) replacement by setting +[var.update_policy](#input_update_policy) to "PROACTIVE". In this case we +recommend the use of Filestore to store the job queue state ("spool") and +setting [var.spool_parent_dir][#input_spool_parent_dir] to its mount point: + +```yaml + - id: spoolfs + source: modules/file-system/filestore + use: + - network1 + settings: + filestore_tier: ENTERPRISE + local_mount: /shared + +... + + - id: htcondor_access + source: community/modules/scheduler/htcondor-access-point + use: + - network1 + - spoolfs + - htcondor_secrets + - htcondor_setup + - htcondor_cm + - htcondor_execute_point_group + settings: + spool_parent_dir: /shared +``` + +[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.1 | +| [google](#requirement\_google) | >= 3.83 | +| [null](#requirement\_null) | >= 3.0 | +| [random](#requirement\_random) | ~> 3.6 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [null](#provider\_null) | >= 3.0 | +| [random](#provider\_random) | ~> 3.6 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [access\_point\_instance\_template](#module\_access\_point\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | +| [htcondor\_ap](#module\_htcondor\_ap) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_compute_address.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | +| [google_compute_disk.spool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [google_compute_region_disk.spool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_region_disk) | resource | +| [google_storage_bucket_object.ap_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [null_resource.ap_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [random_shuffle.zones](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/shuffle) | resource | +| [google_compute_image.htcondor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | +| [google_compute_instance.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance) | data source | +| [google_compute_region_instance_group.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_group) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_point\_runner](#input\_access\_point\_runner) | A list of Toolkit runners for configuring an HTCondor access point | `list(map(string))` | `[]` | no | +| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | Service account for access point (e-mail format) | `string` | n/a | yes | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [autoscaler\_runner](#input\_autoscaler\_runner) | A list of Toolkit runners for configuring autoscaling daemons | `list(map(string))` | `[]` | no | +| [central\_manager\_ips](#input\_central\_manager\_ips) | List of IP addresses of HTCondor Central Managers | `list(string)` | n/a | yes | +| [default\_mig\_id](#input\_default\_mig\_id) | Default MIG ID for HTCondor jobs; if unset, jobs must specify MIG id | `string` | `""` | no | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `32` | no | +| [disk\_type](#input\_disk\_type) | Boot disk size in GB | `string` | `"pd-balanced"` | no | +| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape acoss zones for instance group managing high availability of access point | `string` | `"ANY_SINGLE_ZONE"` | no | +| [enable\_high\_availability](#input\_enable\_high\_availability) | Provision HTCondor access point in high availability mode | `bool` | `false` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [enable\_public\_ips](#input\_enable\_public\_ips) | Enable Public IPs on the access points | `bool` | `false` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | +| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | +| [instance\_image](#input\_instance\_image) | Custom VM image with HTCondor and Toolkit support installed."

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | n/a | yes | +| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | +| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor central managers | `string` | `"n2-standard-4"` | no | +| [metadata](#input\_metadata) | Metadata to add to HTCondor central managers | `map(string)` | `{}` | no | +| [mig\_id](#input\_mig\_id) | List of Managed Instance Group IDs containing execute points in this pool (supplied by htcondor-execute-point module) | `list(string)` | `[]` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network in which the HTCondor central manager will be created. | `string` | `null` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | +| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [spool\_disk\_size\_gb](#input\_spool\_disk\_size\_gb) | Boot disk size in GB | `number` | `32` | no | +| [spool\_disk\_type](#input\_spool\_disk\_type) | Boot disk size in GB | `string` | `"pd-ssd"` | no | +| [spool\_parent\_dir](#input\_spool\_parent\_dir) | HTCondor access point configuration SPOOL will be set to subdirectory named "spool" | `string` | `"/var/lib/condor"` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which the HTCondor central manager will be created. | `string` | `null` | no | +| [update\_policy](#input\_update\_policy) | Replacement policy for Access Point Managed Instance Group ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle) | `string` | `"OPPORTUNISTIC"` | no | +| [zones](#input\_zones) | Zone(s) in which access point may be created. If not supplied, defaults to 2 randomly-selected zones in var.region. | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [access\_point\_ips](#output\_access\_point\_ips) | IP addresses of the access points provisioned by this module | +| [access\_point\_name](#output\_access\_point\_name) | Name of the access point provisioned by this module | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml new file mode 100644 index 0000000000..6a2f50c831 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml @@ -0,0 +1,120 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure HTCondor Access Point + hosts: localhost + become: true + vars: + spool_dir: /var/lib/condor/spool + condor_config_root: /etc/condor + ghpc_config_file: 50-ghpc-managed + htcondor_spool_disk_device: /dev/disk/by-id/google-htcondor-spool-disk + tasks: + - name: Ensure necessary variables are set + ansible.builtin.assert: + that: + - htcondor_role is defined + - config_object is defined + - name: Remove default HTCondor configuration + ansible.builtin.file: + path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" + state: absent + notify: + - Reload HTCondor + - name: Create Toolkit configuration file + register: config_update + changed_when: config_update.rc == 137 + failed_when: config_update.rc != 0 and config_update.rc != 137 + ansible.builtin.shell: | + set -e -o pipefail + REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) + + CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" + if [ -f "${CONFIG_FILE}" ]; then + LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") + else + LOCAL_HASH="INVALID-HASH" + fi + + if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then + gcloud storage cp {{ config_object }} "${CONFIG_FILE}" + chmod 0644 "${CONFIG_FILE}" + exit 137 + fi + args: + executable: /bin/bash + notify: + - Reload HTCondor + - name: Configure HTCondor SchedD + when: htcondor_role == 'get_htcondor_submit' + block: + - name: Format spool disk + community.general.filesystem: + fstype: ext4 + state: present + dev: "{{ htcondor_spool_disk_device }}" + # RUN TUNE2FS + - name: Mount spool (creates mount point) + ansible.posix.mount: + path: "{{ spool_dir }}" + src: "{{ htcondor_spool_disk_device }}" + fstype: ext4 + opts: defaults + state: mounted + - name: Ensure spool free space + ansible.builtin.command: tune2fs -r 0 {{ htcondor_spool_disk_device }} + - name: Setup spool directory + ansible.builtin.file: + path: "{{ spool_dir }}" + state: directory + owner: condor + group: condor + mode: 0755 + recurse: true + - name: Create SystemD override directory for HTCondor + ansible.builtin.file: + path: /etc/systemd/system/condor.service.d + state: directory + owner: root + group: root + mode: 0755 + - name: Ensure HTCondor starts after shared filesystem is mounted + ansible.builtin.copy: + dest: /etc/systemd/system/condor.service.d/mount-spool.conf + mode: 0644 + content: | + [Unit] + RequiresMountsFor={{ spool_dir }} + notify: + - Reload SystemD + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true + - name: Reload HTCondor + ansible.builtin.service: + name: condor + state: reloaded + post_tasks: + - name: Start HTCondor + ansible.builtin.service: + name: condor + state: started + enabled: true + - name: Inform users + changed_when: false + ansible.builtin.shell: | + set -e -o pipefail + wall "******* HTCondor configuration complete; startup-script may still be executing ********" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf new file mode 100644 index 0000000000..fdbcf5c32f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf @@ -0,0 +1,338 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "htcondor-access-point", ghpc_role = "scheduler" }) +} + +locals { + network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } + oslogin_api_values = { + "DISABLE" = "FALSE" + "ENABLE" = "TRUE" + } + enable_oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + metadata = merge( + local.network_storage_metadata, + local.enable_oslogin_metadata, + local.disable_automatic_updates_metadata, + var.metadata + ) + + host_count = 1 + name_prefix = "${var.deployment_name}-ap" + + example_runner = { + type = "data" + destination = "/var/tmp/helloworld.sub" + content = <<-EOT + universe = vanilla + executable = /bin/sleep + arguments = 1000 + output = out.$(ClusterId).$(ProcId) + error = err.$(ClusterId).$(ProcId) + log = log.$(ClusterId).$(ProcId) + request_cpus = 1 + request_memory = 100MB + queue + EOT + } + + native_fstype = [] + startup_script_network_storage = [ + for ns in var.network_storage : + ns if !contains(local.native_fstype, ns.fs_type) + ] + storage_client_install_runners = [ + for ns in local.startup_script_network_storage : + ns.client_install_runner if ns.client_install_runner != null + ] + mount_runners = [ + for ns in local.startup_script_network_storage : + ns.mount_runner if ns.mount_runner != null + ] + + all_runners = concat( + local.storage_client_install_runners, + local.mount_runners, + var.access_point_runner, + [local.schedd_runner], + var.autoscaler_runner, + [local.example_runner] + ) + + ap_config = templatefile("${path.module}/templates/condor_config.tftpl", { + htcondor_role = "get_htcondor_submit", + central_manager_ips = var.central_manager_ips + spool_dir = "${var.spool_parent_dir}/spool", + mig_ids = var.mig_id, + default_mig_id = var.default_mig_id + }) + + ap_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.ap_config.output_name}" + schedd_runner = { + type = "ansible-local" + content = file("${path.module}/files/htcondor_configure.yml") + destination = "htcondor_configure.yml" + args = join(" ", [ + "-e htcondor_role=get_htcondor_submit", + "-e config_object=${local.ap_object}", + "-e spool_dir=${var.spool_parent_dir}/spool", + "-e htcondor_spool_disk_device=/dev/disk/by-id/google-${local.spool_disk_device_name}", + ]) + } + + access_point_ips = google_compute_address.ap.address + access_point_name = data.google_compute_instance.ap.name + + spool_disk_resource_name = "${var.deployment_name}-spool-disk" + spool_disk_device_name = "htcondor-spool-disk" + spool_disk_source = try(google_compute_disk.spool[0].name, google_compute_region_disk.spool[0].self_link) + + zones = coalescelist(var.zones, random_shuffle.zones.result) + + vm_family = split("-", var.machine_type)[0] + regional_pd_families = ["e2", "n1", "n2", "n2d"] +} + +data "google_compute_image" "htcondor" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = var.instance_image.project + + lifecycle { + postcondition { + condition = self.disk_size_gb <= var.disk_size_gb + error_message = "var.disk_size_gb must be set to at least the size of the image (${self.disk_size_gb})" + } + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} + +data "google_compute_zones" "available" { + project = var.project_id + region = var.region + + lifecycle { + postcondition { + condition = alltrue([ + for z in var.zones : contains(self.names, z) + ]) + error_message = "Each entry in var.zones must be a zone in var.region: ${var.region}" + } + } +} + +resource "random_shuffle" "zones" { + input = data.google_compute_zones.available.names + result_count = var.enable_high_availability ? 2 : 1 +} + +data "google_compute_region_instance_group" "ap" { + self_link = module.htcondor_ap.self_link + lifecycle { + postcondition { + condition = length(self.instances) == local.host_count + error_message = "There should be ${local.host_count} access points found" + } + } +} + +data "google_compute_instance" "ap" { + self_link = data.google_compute_region_instance_group.ap.instances[0].instance +} + +resource "null_resource" "ap_config" { + triggers = { + config = local.ap_config + } +} + +resource "google_storage_bucket_object" "ap_config" { + name = "${local.name_prefix}-config-${substr(md5(null_resource.ap_config.id), 0, 4)}" + content = local.ap_config + bucket = var.htcondor_bucket_name + + lifecycle { + precondition { + condition = var.default_mig_id == "" || contains(var.mig_id, var.default_mig_id) + error_message = "If set, var.default_mig_id must be an element in var.mig_id" + } + + # by construction, this precondition only fails when the user has set + # var.zones to a non-empty list of length not equal to 2 + precondition { + condition = !var.enable_high_availability || length(local.zones) == 2 + error_message = "When using HTCondor access point high availability, var.zones must be of length 2." + } + } +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + project_id = var.project_id + region = var.region + labels = local.labels + deployment_name = var.deployment_name + + runners = local.all_runners +} + +resource "google_compute_region_disk" "spool" { + count = var.enable_high_availability ? 1 : 0 + name = local.spool_disk_resource_name + labels = local.labels + type = var.spool_disk_type + region = var.region + size = var.spool_disk_size_gb + + replica_zones = local.zones + + lifecycle { + precondition { + condition = var.spool_disk_size_gb >= 200 + error_message = "When using HTCondor access point high availability, var.spool_disk_size_gb must be set to 200 or greater." + } + + precondition { + condition = contains(local.regional_pd_families, local.vm_family) + error_message = "When using HTCondor access point high availability, var.machine_type must be one of ${jsonencode(local.regional_pd_families)}." + } + } +} + +resource "google_compute_disk" "spool" { + count = var.enable_high_availability ? 0 : 1 + name = local.spool_disk_resource_name + labels = local.labels + type = var.spool_disk_type + zone = local.zones[0] + size = var.spool_disk_size_gb +} + +resource "google_compute_address" "ap" { + project = var.project_id + name = local.name_prefix + region = var.region + subnetwork = var.subnetwork_self_link + address_type = "INTERNAL" + purpose = "GCE_ENDPOINT" +} + +module "access_point_instance_template" { + source = "terraform-google-modules/vm/google//modules/instance_template" + version = "~> 12.1" + + name_prefix = local.name_prefix + project_id = var.project_id + network = var.network_self_link + subnetwork = var.subnetwork_self_link + service_account = { + email = var.access_point_service_account_email + scopes = var.service_account_scopes + } + labels = local.labels + + machine_type = var.machine_type + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + preemptible = false + startup_script = module.startup_script.startup_script + metadata = local.metadata + source_image = data.google_compute_image.htcondor.self_link + + # secure boot + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config + + network_ip = google_compute_address.ap.id + + # spool disk + additional_disks = [ + { + source = local.spool_disk_source + device_name = local.spool_disk_device_name + } + ] +} + +module "htcondor_ap" { + source = "terraform-google-modules/vm/google//modules/mig" + version = "~> 12.1" + + project_id = var.project_id + region = var.region + distribution_policy_target_shape = var.distribution_policy_target_shape + distribution_policy_zones = local.zones + target_size = local.host_count + hostname = local.name_prefix + instance_template = module.access_point_instance_template.self_link + + health_check_name = "health-${local.name_prefix}" + health_check = { + type = "tcp" + initial_delay_sec = 600 + check_interval_sec = 20 + healthy_threshold = 2 + timeout_sec = 8 + unhealthy_threshold = 3 + response = "" + proxy_header = "NONE" + port = 9618 + request = "" + request_path = "" + host = "" + enable_logging = true + } + + update_policy = [{ + instance_redistribution_type = "NONE" + replacement_method = "RECREATE" # preserves hostnames (necessary for PROACTIVE replacement) + max_surge_fixed = 0 # must be 0 to preserve hostnames + max_unavailable_fixed = length(local.zones) + max_surge_percent = null + max_unavailable_percent = null + min_ready_sec = 300 + minimal_action = "REPLACE" + type = var.update_policy + }] + + stateful_disks = [{ + device_name = local.spool_disk_device_name + delete_rule = "ON_PERMANENT_INSTANCE_DELETION" + }] + stateful_ips = var.enable_public_ips ? [{ + interface_name = "nic0" + delete_rule = "ON_PERMANENT_INSTANCE_DELETION" + is_external = true + }] : [] + + # the timeouts below are default for resource + wait_for_instances = true + mig_timeouts = { + create = "15m" + delete = "15m" + update = "15m" + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml new file mode 100644 index 0000000000..3a78f9a46b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - storage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf new file mode 100644 index 0000000000..f7424c6d5d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "access_point_ips" { + description = "IP addresses of the access points provisioned by this module" + value = local.access_point_ips +} + +output "access_point_name" { + description = "Name of the access point provisioned by this module" + value = local.access_point_name +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl new file mode 100644 index 0000000000..214fbc726f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl @@ -0,0 +1,70 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this file is managed by the Cluster Toolkit; do not edit it manually +# override settings with a higher priority (last lexically) named file +# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration + +use role:${htcondor_role} +CONDOR_HOST = ${join(",", central_manager_ips)} + +SPOOL = ${spool_dir} +SCHEDD_INTERVAL = 30 +TRUST_UID_DOMAIN = True +SUBMIT_ATTRS = RunAsOwner +RunAsOwner = True + +# When a job matches to a machine, add machine attributes to the job for +# condor_history (e.g. VM Instance ID) +use feature:JobsHaveInstanceIDs +SYSTEM_JOB_MACHINE_ATTRS = $(SYSTEM_JOB_MACHINE_ATTRS) \ + CloudVMType CloudZone CloudInterruptible +SYSTEM_JOB_MACHINE_ATTRS_HISTORY_LENGTH = 10 + +# Add Cloud attributes to SchedD ClassAd +use feature:ScheddCronOneShot(cloud, $(LIBEXEC)/common-cloud-attributes-google.py) +SCHEDD_CRON_cloud_PREFIX = Cloud + +# aid the user by automatically using RequireSpot in their Requirements, unless +# the user has explicitly used CloudInterruptible +JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) SPOT +JOB_TRANSFORM_SPOT @=end + REQUIREMENTS ! isUndefined(RequireSpot) && ! unresolved(Requirements, "^CloudInterruptible$") + SET Requirements ($(MY.Requirements)) && (CloudInterruptible is My.RequireSpot) +@end + +# help the user by enforcing that RequireSpot is undefined or a boolean +SUBMIT_REQUIREMENT_NAMES = $(SUBMIT_REQUIREMENT_NAMES) SPOT +SUBMIT_REQUIREMENT_SPOT = isUndefined(RequireSpot) || isBoolean(RequireSpot) +SUBMIT_REQUIREMENT_SPOT_REASON = "If +RequireSpot is defined, it must be either True or False" + +%{ if length(mig_ids) > 0 ~} +MIG_IDS = "${join(" ", mig_ids)}" +MIG_ID_LIST = split($(MIG_IDS)) +%{ if default_mig_id != "" ~} +JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) ID_DEFAULT +JOB_TRANSFORM_ID_DEFAULT @=end + DEFAULT RequireId "${default_mig_id}" +@end +%{ endif ~} +SUBMIT_REQUIREMENT_NAMES = $(SUBMIT_REQUIREMENT_NAMES) MIGID +SUBMIT_REQUIREMENT_MIGID = !isUndefined(RequireId) && member(RequireId, $(MIG_ID_LIST)) +SUBMIT_REQUIREMENT_MIGID_REASON = strcat("Jobs must set +RequireId to one of following values surrounded by quotation marks:\n", $(MIG_IDS)) + +JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) MIGID +JOB_TRANSFORM_MIGID @=end + REQUIREMENTS ! isUndefined(RequireId) && ! unresolved(Requirements, "^CloudCreatedBy$") + SET Requirements ($(MY.Requirements)) && regexp(strcat("/", My.RequireId, "$"), CloudCreatedBy) +@end +%{ endif ~} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf new file mode 100644 index 0000000000..f54a88ac2e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf @@ -0,0 +1,266 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which HTCondor pool will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "labels" { + description = "Labels to add to resources. List key, value pairs." + type = map(string) +} + +variable "region" { + description = "Default region for creating resources" + type = string +} + +variable "zones" { + description = "Zone(s) in which access point may be created. If not supplied, defaults to 2 randomly-selected zones in var.region." + type = list(string) + default = [] + nullable = false + + validation { + condition = length(var.zones) <= 2 + error_message = "Set var.zones to the empty list or up to 2 zones in var.region" + } +} + +variable "distribution_policy_target_shape" { + description = "Target shape acoss zones for instance group managing high availability of access point" + type = string + default = "ANY_SINGLE_ZONE" +} + +variable "network_self_link" { + description = "The self link of the network in which the HTCondor central manager will be created." + type = string + default = null +} + +variable "access_point_service_account_email" { + description = "Service account for access point (e-mail format)" + type = string +} + +variable "service_account_scopes" { + description = "Scopes by which to limit service account attached to central manager." + type = set(string) + default = [ + "https://www.googleapis.com/auth/cloud-platform", + ] +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured" + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "disk_size_gb" { + description = "Boot disk size in GB" + type = number + default = 32 + nullable = false +} + +variable "disk_type" { + description = "Boot disk size in GB" + type = string + default = "pd-balanced" + nullable = false +} + +variable "spool_disk_size_gb" { + description = "Boot disk size in GB" + type = number + default = 32 + nullable = false +} + +variable "spool_disk_type" { + description = "Boot disk size in GB" + type = string + default = "pd-ssd" + nullable = false +} + +variable "metadata" { + description = "Metadata to add to HTCondor central managers" + type = map(string) + default = {} +} + +variable "enable_oslogin" { + description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." + type = string + default = "ENABLE" + nullable = false + validation { + condition = contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) + error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." + } +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork in which the HTCondor central manager will be created." + type = string + default = null +} + +variable "enable_high_availability" { + description = "Provision HTCondor access point in high availability mode" + type = bool + default = false +} + +variable "instance_image" { + description = <<-EOD + Custom VM image with HTCondor and Toolkit support installed." + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "machine_type" { + description = "Machine type to use for HTCondor central managers" + type = string + default = "n2-standard-4" +} + +variable "access_point_runner" { + description = "A list of Toolkit runners for configuring an HTCondor access point" + type = list(map(string)) + default = [] +} + +variable "autoscaler_runner" { + description = "A list of Toolkit runners for configuring autoscaling daemons" + type = list(map(string)) + default = [] +} + +variable "spool_parent_dir" { + description = "HTCondor access point configuration SPOOL will be set to subdirectory named \"spool\"" + type = string + default = "/var/lib/condor" +} + +variable "central_manager_ips" { + description = "List of IP addresses of HTCondor Central Managers" + type = list(string) +} + +variable "htcondor_bucket_name" { + description = "Name of HTCondor configuration bucket" + type = string +} + +variable "enable_public_ips" { + description = "Enable Public IPs on the access points" + type = bool + default = false +} + +variable "mig_id" { + description = "List of Managed Instance Group IDs containing execute points in this pool (supplied by htcondor-execute-point module)" + type = list(string) + default = [] + nullable = false + + validation { + condition = length(var.mig_id) > 0 + error_message = "At least 1 MIG containing execute points must be provided to this module" + } +} + +variable "default_mig_id" { + description = "Default MIG ID for HTCondor jobs; if unset, jobs must specify MIG id" + type = string + default = "" + nullable = false +} + +variable "enable_shielded_vm" { + type = bool + default = false + description = "Enable the Shielded VM configuration (var.shielded_instance_config)." +} + +variable "shielded_instance_config" { + description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" + type = object({ + enable_secure_boot = bool + enable_vtpm = bool + enable_integrity_monitoring = bool + }) + + default = { + enable_secure_boot = true + enable_vtpm = true + enable_integrity_monitoring = true + } +} + +variable "update_policy" { + description = "Replacement policy for Access Point Managed Instance Group (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)" + type = string + default = "OPPORTUNISTIC" + validation { + condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) + error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf new file mode 100644 index 0000000000..e64942c840 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:htcondor-access-point/v1.57.0" + } + + required_version = ">= 1.1" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md new file mode 100644 index 0000000000..dfab563a55 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md @@ -0,0 +1,159 @@ +## Description + +This module provisions a highly available HTCondor central manager using a [Managed +Instance Group (MIG)][mig] with auto-healing. + +[mig]: https://cloud.google.com/compute/docs/instance-groups + +## Usage + +This module provisions an HTCondor central manager with a standard +configuration. For the node to function correctly, you must supply the input +variable described below: + +- [var.central_manager_runner](#input_central_manager_runner) + - Runner must download a POOL password / signing key and create an [IDTOKEN] + with no scopes (full authorization). + +A reference implementation is included in the Toolkit module +[htcondor-pool-secrets]. You may substitute implementations so long as they +duplicate the functionality in the references. Usage is demonstrated in the +[HTCondor example][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- +[htcondor-pool-secrets]: ../htcondor-pool-secrets/README.md +[IDTOKEN]: https://htcondor.readthedocs.io/en/latest/admin-manual/security.html#introducing-idtokens + +## Behavior of Managed Instance Group (MIG) + +A regional [MIG][mig] is used to provision the central manager, although only +1 node will ever be active at a time. By default, the node will be provisioned +in any of the zones available in that region, however, it can be constrained to +run in fewer zones (or a single zone) using [var.zones](#input_zones). + +When the configuration of the Central Manager is changed, the MIG can be +configured to [replace the VM][replacement] using a "proactive" or +"opportunistic" policy. By default, the Central Manager replacement policy is +set to proactive. In practice, this means that the Central Manager will be +replaced by Terraform when changes to the instance template / HTCondor +configuration are made. The Central Manager is safe to replace automatically as +it gathers its state information from periodic messages exchanged with the rest +of the HTCondor pool. + +This mode can be configured by setting [var.update_policy](#input_update_policy) +to either "PROACTIVE" (default) or "OPPORTUNISTIC". If set to opportunistic +replacement, the Central Manager will be replaced only when: + +- intentionally by issuing an update via Cloud Console or using gcloud (below) +- the VM becomes unhealthy or is otherwise automatically replaced (e.g. regular + Google Cloud maintenance) + +For example, to manually update all instances in a MIG: + +```text +gcloud compute instance-groups managed update-instances \ + <> --all-instances --region <> \ + --project <> --minimal-action replace +``` + +[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type + +## Limiting inter-zone egress + +Because all the elements of the HTCondor pool use regional MIGs, they may be +subject to [interzone egress fees][network-pricing]. The primary traffic between +nodes of an HTCondor pool running embarrassingly parallel jobs is expected to +be limited to API traffic for job scheduling and monitoring. Please review the +[network pricing][network-pricing] documentation and determine if this cost is +a concern. If it is, use [var.zones](#input_zones) to constrain each node within +your HTCondor pool to operate within a single zone. + +[network-pricing]: https://cloud.google.com/vpc/network-pricing + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.1.0 | +| [google](#requirement\_google) | >= 3.83 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [null](#provider\_null) | >= 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [central\_manager\_instance\_template](#module\_central\_manager\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | +| [htcondor\_cm](#module\_htcondor\_cm) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_compute_address.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | +| [google_storage_bucket_object.cm_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [null_resource.cm_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_image.htcondor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | +| [google_compute_instance.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance) | data source | +| [google_compute_region_instance_group.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_group) | data source | +| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [central\_manager\_runner](#input\_central\_manager\_runner) | A list of Toolkit runners for configuring an HTCondor central manager | `list(map(string))` | `[]` | no | +| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | Service account e-mail for central manager (can be supplied by htcondor-setup module) | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `20` | no | +| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape for instance group managing high availability of central manager | `string` | `"ANY_SINGLE_ZONE"` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | +| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | +| [instance\_image](#input\_instance\_image) | Custom VM image with HTCondor installed using the htcondor-install module."

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | n/a | yes | +| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | +| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor central managers | `string` | `"n2-standard-4"` | no | +| [metadata](#input\_metadata) | Metadata to add to HTCondor central managers | `map(string)` | `{}` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network in which the HTCondor central manager will be created. | `string` | `null` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | Project in which HTCondor central manager will be created | `string` | n/a | yes | +| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which the HTCondor central manager will be created. | `string` | `null` | no | +| [update\_policy](#input\_update\_policy) | Replacement policy for Central Manager ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle). | `string` | `"PROACTIVE"` | no | +| [zones](#input\_zones) | Zone(s) in which central manager may be created. If not supplied, will default to all zones in var.region. | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [central\_manager\_ips](#output\_central\_manager\_ips) | IP addresses of the central managers provisioned by this module | +| [central\_manager\_name](#output\_central\_manager\_name) | Name of the central managers provisioned by this module | +| [list\_instances\_command](#output\_list\_instances\_command) | Command to list central managers provisioned by this module | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml new file mode 100644 index 0000000000..7408af6370 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml @@ -0,0 +1,72 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure HTCondor central manager + hosts: localhost + become: true + vars: + condor_config_root: /etc/condor + ghpc_config_file: 50-ghpc-managed + tasks: + - name: Ensure necessary variables are set + ansible.builtin.assert: + that: + - config_object is defined + - name: Remove default HTCondor configuration + ansible.builtin.file: + path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" + state: absent + notify: + - Reload HTCondor + - name: Create Toolkit configuration file + register: config_update + changed_when: config_update.rc == 137 + failed_when: config_update.rc != 0 and config_update.rc != 137 + ansible.builtin.shell: | + set -e -o pipefail + REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) + + CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" + if [ -f "${CONFIG_FILE}" ]; then + LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") + else + LOCAL_HASH="INVALID-HASH" + fi + + if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then + gcloud storage cp {{ config_object }} "${CONFIG_FILE}" + chmod 0644 "${CONFIG_FILE}" + exit 137 + fi + args: + executable: /bin/bash + notify: + - Reload HTCondor + handlers: + - name: Reload HTCondor + ansible.builtin.service: + name: condor + state: reloaded + post_tasks: + - name: Start HTCondor + ansible.builtin.service: + name: condor + state: started + enabled: true + - name: Inform users + changed_when: false + ansible.builtin.shell: | + set -e -o pipefail + wall "******* HTCondor system configuration complete ********" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf new file mode 100644 index 0000000000..d288a91144 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf @@ -0,0 +1,226 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "htcondor-central-manager", ghpc_role = "scheduler" }) +} + +locals { + network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } + oslogin_api_values = { + "DISABLE" = "FALSE" + "ENABLE" = "TRUE" + } + enable_oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + metadata = merge( + local.network_storage_metadata, + local.enable_oslogin_metadata, + local.disable_automatic_updates_metadata, + var.metadata + ) + + name_prefix = "${var.deployment_name}-cm" + + cm_config = templatefile("${path.module}/templates/condor_config.tftpl", {}) + + cm_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.cm_config.output_name}" + schedd_runner = { + type = "ansible-local" + content = file("${path.module}/files/htcondor_configure.yml") + destination = "htcondor_configure.yml" + args = join(" ", [ + "-e config_object=${local.cm_object}", + ]) + } + + native_fstype = [] + startup_script_network_storage = [ + for ns in var.network_storage : + ns if !contains(local.native_fstype, ns.fs_type) + ] + storage_client_install_runners = [ + for ns in local.startup_script_network_storage : + ns.client_install_runner if ns.client_install_runner != null + ] + mount_runners = [ + for ns in local.startup_script_network_storage : + ns.mount_runner if ns.mount_runner != null + ] + + all_runners = concat( + local.storage_client_install_runners, + local.mount_runners, + var.central_manager_runner, + [local.schedd_runner] + ) + + central_manager_ips = google_compute_address.cm.address + central_manager_name = data.google_compute_instance.cm.name + + list_instances_command = "gcloud compute instance-groups list-instances ${data.google_compute_region_instance_group.cm.name} --region ${var.region} --project ${var.project_id}" + + zones = coalescelist(var.zones, data.google_compute_zones.available.names) +} + +data "google_compute_image" "htcondor" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = var.instance_image.project + + lifecycle { + postcondition { + condition = self.disk_size_gb <= var.disk_size_gb + error_message = "var.disk_size_gb must be set to at least the size of the image (${self.disk_size_gb})" + } + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} + +data "google_compute_zones" "available" { + project = var.project_id + region = var.region +} + +data "google_compute_region_instance_group" "cm" { + self_link = module.htcondor_cm.self_link + lifecycle { + postcondition { + condition = length(self.instances) == 1 + error_message = "There should only be 1 central manager found" + } + } +} + +data "google_compute_instance" "cm" { + self_link = data.google_compute_region_instance_group.cm.instances[0].instance +} + +resource "null_resource" "cm_config" { + triggers = { + config = local.cm_config + } +} + +resource "google_storage_bucket_object" "cm_config" { + name = "${local.name_prefix}-config-${substr(md5(null_resource.cm_config.id), 0, 4)}" + content = local.cm_config + bucket = var.htcondor_bucket_name +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + project_id = var.project_id + region = var.region + labels = local.labels + deployment_name = var.deployment_name + + runners = local.all_runners +} + +resource "google_compute_address" "cm" { + project = var.project_id + name = local.name_prefix + region = var.region + subnetwork = var.subnetwork_self_link + address_type = "INTERNAL" + purpose = "GCE_ENDPOINT" +} + +module "central_manager_instance_template" { + source = "terraform-google-modules/vm/google//modules/instance_template" + version = "~> 12.1" + + name_prefix = local.name_prefix + project_id = var.project_id + network = var.network_self_link + subnetwork = var.subnetwork_self_link + service_account = { + email = var.central_manager_service_account_email + scopes = var.service_account_scopes + } + labels = local.labels + + machine_type = var.machine_type + disk_size_gb = var.disk_size_gb + preemptible = false + startup_script = module.startup_script.startup_script + metadata = local.metadata + source_image = data.google_compute_image.htcondor.self_link + + # secure boot + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config + + network_ip = google_compute_address.cm.id +} + +module "htcondor_cm" { + source = "terraform-google-modules/vm/google//modules/mig" + version = "~> 12.1" + + project_id = var.project_id + region = var.region + distribution_policy_target_shape = var.distribution_policy_target_shape + distribution_policy_zones = local.zones + target_size = 1 + hostname = local.name_prefix + instance_template = module.central_manager_instance_template.self_link + + health_check_name = "health-${local.name_prefix}" + health_check = { + type = "tcp" + initial_delay_sec = 600 + check_interval_sec = 20 + healthy_threshold = 2 + timeout_sec = 8 + unhealthy_threshold = 3 + response = "" + proxy_header = "NONE" + port = 9618 + request = "" + request_path = "" + host = "" + enable_logging = true + } + + update_policy = [{ + instance_redistribution_type = "NONE" + replacement_method = "RECREATE" # preserves hostnames (necessary for PROACTIVE replacement) + max_surge_fixed = 0 # must be 0 to preserve hostnames + max_unavailable_fixed = length(local.zones) + max_surge_percent = null + max_unavailable_percent = null + min_ready_sec = 300 + minimal_action = "REPLACE" + type = var.update_policy + }] + + # the timeouts below are default for resource + wait_for_instances = true + mig_timeouts = { + create = "15m" + delete = "15m" + update = "15m" + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml new file mode 100644 index 0000000000..3a78f9a46b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - storage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf new file mode 100644 index 0000000000..a6272e7ca2 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "list_instances_command" { + description = "Command to list central managers provisioned by this module" + value = local.list_instances_command +} + +output "central_manager_ips" { + description = "IP addresses of the central managers provisioned by this module" + value = local.central_manager_ips +} + +output "central_manager_name" { + description = "Name of the central managers provisioned by this module" + value = local.central_manager_name +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl new file mode 100644 index 0000000000..5b9676457e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl @@ -0,0 +1,31 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this file is managed by the Cluster Toolkit; do not edit it manually +# override settings with a higher priority (last lexically) named file +# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration + +use role:get_htcondor_central_manager +CONDOR_HOST = $(IPV4_ADDRESS) + +# Central Manager configuration settings +# https://htcondor.readthedocs.io/en/23.0/admin-manual/configuration-macros.html#condor-collector-configuration-file-entries +# https://htcondor.readthedocs.io/en/23.0/admin-manual/configuration-macros.html#condor-negotiator-configuration-file-entries +# set classad lifetime (expiration) to ~5x the update interval for all daemons +# defaults to 900s +CLASSAD_LIFETIME = 180 +COLLECTOR_UPDATE_INTERVAL = 30 +NEGOTIATOR_UPDATE_INTERVAL = 30 +NEGOTIATOR_DEPTH_FIRST = True +NEGOTIATOR_UPDATE_AFTER_CYCLE = True diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf new file mode 100644 index 0000000000..7f85861c3f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf @@ -0,0 +1,192 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which HTCondor central manager will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "labels" { + description = "Labels to add to resources. List key, value pairs." + type = map(string) +} + +variable "region" { + description = "Default region for creating resources" + type = string +} + +variable "zones" { + description = "Zone(s) in which central manager may be created. If not supplied, will default to all zones in var.region." + type = list(string) + default = [] + nullable = false +} + +variable "distribution_policy_target_shape" { + description = "Target shape for instance group managing high availability of central manager" + type = string + default = "ANY_SINGLE_ZONE" +} + +variable "network_self_link" { + description = "The self link of the network in which the HTCondor central manager will be created." + type = string + default = null +} + +variable "central_manager_service_account_email" { + description = "Service account e-mail for central manager (can be supplied by htcondor-setup module)" + type = string +} + +variable "service_account_scopes" { + description = "Scopes by which to limit service account attached to central manager." + type = set(string) + default = [ + "https://www.googleapis.com/auth/cloud-platform", + ] +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured" + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "disk_size_gb" { + description = "Boot disk size in GB" + type = number + default = 20 + nullable = false +} + +variable "metadata" { + description = "Metadata to add to HTCondor central managers" + type = map(string) + default = {} +} + +variable "enable_oslogin" { + description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." + type = string + default = "ENABLE" + nullable = false + validation { + condition = contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) + error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." + } +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork in which the HTCondor central manager will be created." + type = string + default = null +} + +variable "instance_image" { + description = <<-EOD + Custom VM image with HTCondor installed using the htcondor-install module." + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "machine_type" { + description = "Machine type to use for HTCondor central managers" + type = string + default = "n2-standard-4" +} + +variable "central_manager_runner" { + description = "A list of Toolkit runners for configuring an HTCondor central manager" + type = list(map(string)) + default = [] +} + +variable "htcondor_bucket_name" { + description = "Name of HTCondor configuration bucket" + type = string +} + +variable "enable_shielded_vm" { + type = bool + default = false + description = "Enable the Shielded VM configuration (var.shielded_instance_config)." +} + +variable "shielded_instance_config" { + description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" + type = object({ + enable_secure_boot = bool + enable_vtpm = bool + enable_integrity_monitoring = bool + }) + + default = { + enable_secure_boot = true + enable_vtpm = true + enable_integrity_monitoring = true + } +} + +variable "update_policy" { + description = "Replacement policy for Central Manager (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)." + type = string + default = "PROACTIVE" + validation { + condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) + error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf new file mode 100644 index 0000000000..24051328ff --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:htcondor-central-manager/v1.57.0" + } + + required_version = ">= 1.1.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md new file mode 100644 index 0000000000..7158e7bac6 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md @@ -0,0 +1,172 @@ +## Description + +This module is responsible for the following actions: + +- store an HTCondor Pool password in Google Cloud Secret Manager + - will generate a new password if one is not supplied +- create a secret in Google Cloud Secret Manager in which the HTCondor central + manager can place IDTOKENs (JWT Authorizations) for execute points to download +- create a Toolkit runner for the central manager + - download the POOL password / signing key + - create a local IDTOKEN for itself + - upload the execute point IDTOKEN secret +- create a Toolkit runner for access points + - download the POOL password / signing key + - create a local IDTOKEN for itself +- create a Toolkit runner for execute points + - Fetch the IDTOKEN secret generated by the central manager + +It is expected to be used with the [htcondor-install] and +[htcondor-execute-point] modules. + +[hpcvmimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm +[htcondor-install]: ../../scripts/htcondor-setup/README.md +[htcondor-execute-point]: ../../compute/htcondor-execute-point/README.md + +[htcrole]: https://htcondor.readthedocs.io/en/latest/getting-htcondor/admin-quick-start.html#what-get-htcondor-does-to-configure-a-role + +### Example + +The following code snippet uses this module to create a startup script that +installs HTCondor software and configures an HTCondor Central Manager. A full +example can be found in the [examples README][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- + +```yaml +- id: network1 + source: modules/network/pre-existing-vpc + +- id: htcondor_install + source: community/modules/scripts/htcondor-install + +- id: htcondor_setup + source: community/modules/scheduler/htcondor-setup + use: + - network1 + +- id: htcondor_secrets + source: community/modules/scheduler/htcondor-pool-secrets + use: + - htcondor_setup + + - id: htcondor_startup_central_manager + source: modules/scripts/startup-script + settings: + runners: + - $(htcondor_install.install_htcondor_runner) + - $(htcondor_secrets.central_manager_runner) + - $(htcondor_setup.central_manager_runner) + +- id: htcondor_cm + source: modules/compute/vm-instance + use: + - network1 + - htcondor_startup_central_manager + settings: + name_prefix: cm0 + machine_type: c2-standard-4 + disable_public_ips: true + service_account: + email: $(htcondor_setup.central_manager_service_account) + scopes: + - cloud-platform + network_interfaces: + - network: null + subnetwork: $(network1.subnetwork_self_link) + subnetwork_project: $(vars.project_id) + network_ip: $(htcondor_setup.central_manager_internal_ip) + stack_type: null + access_config: [] + ipv6_access_config: [] + alias_ip_range: [] + nic_type: VIRTIO_NET + queue_count: null + outputs: + - internal_ip +``` + +## Support + +HTCondor is maintained by the [Center for High Throughput Computing][chtc] at +the University of Wisconsin-Madison. Support for HTCondor is available via: + +- [Discussion lists](https://htcondor.org/mail-lists/) +- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) +- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) + +[chtc]: https://chtc.cs.wisc.edu/ + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [google](#requirement\_google) | >= 4.84 | +| [random](#requirement\_random) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.84 | +| [random](#provider\_random) | >= 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_secret_manager_secret.execute_point_idtoken](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | +| [google_secret_manager_secret.pool_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | +| [google_secret_manager_secret_iam_member.access_point](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | +| [google_secret_manager_secret_iam_member.central_manager_idtoken](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | +| [google_secret_manager_secret_iam_member.central_manager_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | +| [google_secret_manager_secret_iam_member.execute_point](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | +| [google_secret_manager_secret_version.pool_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | +| [random_password.pool](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | +| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | +| [pool\_password](#input\_pool\_password) | HTCondor Pool Password | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | +| [trust\_domain](#input\_trust\_domain) | Trust domain for HTCondor pool (if not supplied, will be set based on project\_id) | `string` | `""` | no | +| [user\_managed\_replication](#input\_user\_managed\_replication) | Replication parameters that will be used for defined secrets | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [access\_point\_runner](#output\_access\_point\_runner) | Toolkit Runner to download pool secrets to an HTCondor access point | +| [central\_manager\_runner](#output\_central\_manager\_runner) | Toolkit Runner to download pool secrets to an HTCondor central manager | +| [execute\_point\_runner](#output\_execute\_point\_runner) | Toolkit Runner to download pool secrets to an HTCondor execute point | +| [pool\_password\_secret\_id](#output\_pool\_password\_secret\_id) | Google Cloud Secret Manager ID containing HTCondor Pool Password | +| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | PowerShell script to download pool secrets to an HTCondor execute point | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml new file mode 100644 index 0000000000..538c809c2a --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml @@ -0,0 +1,102 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure HTCondor Secrets + hosts: localhost + become: true + vars: + condor_config_root: /etc/condor + tasks: + - name: Ensure necessary variables are set + ansible.builtin.assert: + that: + - htcondor_role is defined + - password_id is defined + - trust_domain is defined + - name: Set Pool Trust Domain + ansible.builtin.copy: + dest: "{{ condor_config_root }}/config.d/51-ghpc-trust-domain" + mode: 0644 + content: | + # these lines must appear AFTER any "use role:" settings + UID_DOMAIN = {{ trust_domain }} + TRUST_DOMAIN = {{ trust_domain }} + - name: Get HTCondor Pool password (token signing key) + when: htcondor_role != 'get_htcondor_execute' + ansible.builtin.shell: | + set -e -o pipefail +o history + POOL_PASSWORD=$(gcloud secrets versions access latest --secret={{ password_id }}) + echo -n "$POOL_PASSWORD" | sh -c "condor_store_cred add -c -i -" + args: + creates: "{{ condor_config_root }}/passwords.d/POOL" + executable: /bin/bash + - name: Configure HTCondor Central Manager + when: htcondor_role == 'get_htcondor_central_manager' + block: + - name: Create IDTOKEN for Central Manager + ansible.builtin.shell: | + umask 0077 + condor_token_create -identity condor@{{ trust_domain }} \ + -token condor@{{ trust_domain }} + args: + creates: "{{ condor_config_root }}/tokens.d/condor@{{ trust_domain }}" + - name: Create IDTOKEN secret for Execute Points + when: xp_idtoken_secret_id | length > 0 + changed_when: true + ansible.builtin.shell: | + umask 0077 + TMPFILE=$(mktemp) + condor_token_create -authz READ -authz ADVERTISE_MASTER \ + -authz ADVERTISE_STARTD -identity condor@{{ trust_domain }} > "$TMPFILE" + gcloud secrets versions add --data-file "$TMPFILE" {{ xp_idtoken_secret_id }} + rm -f "$TMPFILE" + - name: Configure HTCondor SchedD + when: htcondor_role == 'get_htcondor_submit' + block: + - name: Create IDTOKEN to advertise access point + ansible.builtin.shell: | + umask 0077 + # DAEMON authorization can likely be removed in future when scopes + # needed to trigger a negotiation cycle are changed. Suggest review + # https://opensciencegrid.atlassian.net/jira/software/c/projects/HTCONDOR/issues/?filter=allissues + condor_token_create -authz READ -authz ADVERTISE_MASTER \ + -authz ADVERTISE_SCHEDD -authz DAEMON -identity condor@{{ trust_domain }} \ + -token condor@{{ trust_domain }} + args: + creates: "{{ condor_config_root }}/tokens.d/condor@{{ trust_domain }}" + - name: Configure HTCondor StartD + when: htcondor_role == 'get_htcondor_execute' + block: + - name: Create SystemD override directory for HTCondor Execute Point + ansible.builtin.file: + path: /etc/systemd/system/condor.service.d + state: directory + owner: root + group: root + mode: 0755 + - name: Fetch IDTOKEN to advertise execute point + ansible.builtin.copy: + dest: "/etc/systemd/system/condor.service.d/htcondor-token-fetcher.conf" + mode: 0644 + content: | + [Service] + ExecStartPre=gcloud secrets versions access latest --secret {{ xp_idtoken_secret_id }} \ + --out-file {{ condor_config_root }}/tokens.d/condor@{{ trust_domain }} + notify: + - Reload SystemD + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf new file mode 100644 index 0000000000..1a7c761760 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf @@ -0,0 +1,168 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "htcondor-pool-secrets", ghpc_role = "scheduler" }) +} + +locals { + pool_password = coalesce(var.pool_password, random_password.pool.result) + auto = length(var.user_managed_replication) == 0 ? "" : "-user" + access_point_service_account_iam_email = "serviceAccount:${var.access_point_service_account_email}" + central_manager_service_account_iam_email = "serviceAccount:${var.central_manager_service_account_email}" + execute_point_service_account_iam_email = "serviceAccount:${var.execute_point_service_account_email}" + + trust_domain = coalesce(var.trust_domain, "c.${var.project_id}.internal") + + runner_cm = { + "type" = "ansible-local" + "content" = file("${path.module}/files/htcondor_secrets.yml") + "destination" = "htcondor_secrets.yml" + "args" = join(" ", [ + "-e htcondor_role=get_htcondor_central_manager", + "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", + "-e xp_idtoken_secret_id=${google_secret_manager_secret.execute_point_idtoken.secret_id}", + "-e trust_domain=${local.trust_domain}", + ]) + } + + runner_access = { + "type" = "ansible-local" + "content" = file("${path.module}/files/htcondor_secrets.yml") + "destination" = "htcondor_secrets.yml" + "args" = join(" ", [ + "-e htcondor_role=get_htcondor_submit", + "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", + "-e trust_domain=${local.trust_domain}", + ]) + } + + runner_execute = { + "type" = "ansible-local" + "content" = file("${path.module}/files/htcondor_secrets.yml") + "destination" = "htcondor_secrets.yml" + "args" = join(" ", [ + "-e htcondor_role=get_htcondor_execute", + "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", + "-e xp_idtoken_secret_id=${google_secret_manager_secret.execute_point_idtoken.secret_id}", + "-e trust_domain=${local.trust_domain}", + ]) + } + windows_startup_ps1 = templatefile( + "${path.module}/templates/fetch-idtoken.ps1.tftpl", + { + trust_domain = local.trust_domain, + xp_idtoken_secret_id = google_secret_manager_secret.execute_point_idtoken.secret_id, + } + ) +} + +resource "random_password" "pool" { + length = 24 + special = true + override_special = "_-#=." +} + +resource "google_secret_manager_secret" "pool_password" { + secret_id = "${var.deployment_name}-pool-password${local.auto}" + + labels = local.labels + + replication { + dynamic "auto" { + for_each = length(var.user_managed_replication) == 0 ? [1] : [] + content {} + } + dynamic "user_managed" { + for_each = length(var.user_managed_replication) == 0 ? [] : [1] + content { + dynamic "replicas" { + for_each = var.user_managed_replication + content { + location = replicas.value.location + dynamic "customer_managed_encryption" { + for_each = compact([replicas.value.kms_key_name]) + content { + kms_key_name = customer_managed_encryption.value + } + } + } + } + } + } + } +} + +resource "google_secret_manager_secret_version" "pool_password" { + secret = google_secret_manager_secret.pool_password.id + secret_data = local.pool_password +} + +# this secret will be populated by the Central Manager +resource "google_secret_manager_secret" "execute_point_idtoken" { + secret_id = "${var.deployment_name}-execute-point-idtoken${local.auto}" + + labels = local.labels + + replication { + dynamic "auto" { + for_each = length(var.user_managed_replication) == 0 ? [1] : [] + content {} + } + dynamic "user_managed" { + for_each = length(var.user_managed_replication) == 0 ? [] : [1] + content { + dynamic "replicas" { + for_each = var.user_managed_replication + content { + location = replicas.value.location + dynamic "customer_managed_encryption" { + for_each = compact([replicas.value.kms_key_name]) + content { + kms_key_name = customer_managed_encryption.value + } + } + } + } + } + } + } +} + +resource "google_secret_manager_secret_iam_member" "central_manager_password" { + secret_id = google_secret_manager_secret.pool_password.id + role = "roles/secretmanager.secretAccessor" + member = local.central_manager_service_account_iam_email +} + +resource "google_secret_manager_secret_iam_member" "central_manager_idtoken" { + secret_id = google_secret_manager_secret.execute_point_idtoken.id + role = "roles/secretmanager.secretVersionManager" + member = local.central_manager_service_account_iam_email +} + +resource "google_secret_manager_secret_iam_member" "access_point" { + secret_id = google_secret_manager_secret.pool_password.id + role = "roles/secretmanager.secretAccessor" + member = local.access_point_service_account_iam_email +} + +resource "google_secret_manager_secret_iam_member" "execute_point" { + secret_id = google_secret_manager_secret.execute_point_idtoken.id + role = "roles/secretmanager.secretAccessor" + member = local.execute_point_service_account_iam_email +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml new file mode 100644 index 0000000000..4b0bdbd616 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml @@ -0,0 +1,20 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - iam.googleapis.com + - secretmanager.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf new file mode 100644 index 0000000000..81c4986b16 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "pool_password_secret_id" { + description = "Google Cloud Secret Manager ID containing HTCondor Pool Password" + value = google_secret_manager_secret.pool_password.secret_id + sensitive = true +} + +output "central_manager_runner" { + description = "Toolkit Runner to download pool secrets to an HTCondor central manager" + value = local.runner_cm + depends_on = [ + google_secret_manager_secret_version.pool_password + ] +} + +output "access_point_runner" { + description = "Toolkit Runner to download pool secrets to an HTCondor access point" + value = local.runner_access + depends_on = [ + google_secret_manager_secret_version.pool_password + ] +} + +output "execute_point_runner" { + description = "Toolkit Runner to download pool secrets to an HTCondor execute point" + value = local.runner_execute + depends_on = [ + google_secret_manager_secret_version.pool_password + ] +} + +output "windows_startup_ps1" { + description = "PowerShell script to download pool secrets to an HTCondor execute point" + value = local.windows_startup_ps1 +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl new file mode 100644 index 0000000000..04c96291ee --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl @@ -0,0 +1,26 @@ +Set-StrictMode -Version latest +$ErrorActionPreference = 'Stop' + +$config_dir = 'C:\Condor\config' +if(!(test-path -PathType container -Path $config_dir)) +{ + New-Item -ItemType Directory -Path $config_dir +} +$config_file = "$config_dir\51-ghpc-trust-domain" + +$config_string = @' +# these lines must appear AFTER any "use role:" settings +UID_DOMAIN = ${trust_domain} +TRUST_DOMAIN = ${trust_domain} +'@ + +Set-Content -Path "$config_file" -Value "$config_string" + +# obtain IDTOKEN for authentication by StartD to Central Manager +gcloud secrets versions access latest --secret ${xp_idtoken_secret_id} ` + --out-file C:\condor\tokens.d\condor@${trust_domain} + +if ($LASTEXITCODE -ne 0) +{ + throw "Could not download HTCondor IDTOKEN; exiting startup script" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf new file mode 100644 index 0000000000..22ef3644e8 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf @@ -0,0 +1,67 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which HTCondor pool will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "labels" { + description = "Labels to add to resources. List key, value pairs." + type = map(string) +} + +variable "access_point_service_account_email" { + description = "HTCondor access point service account e-mail" + type = string +} + +variable "central_manager_service_account_email" { + description = "HTCondor access point service account e-mail" + type = string +} + +variable "execute_point_service_account_email" { + description = "HTCondor access point service account e-mail" + type = string +} + +variable "pool_password" { + description = "HTCondor Pool Password" + type = string + sensitive = true + default = null +} + +variable "trust_domain" { + description = "Trust domain for HTCondor pool (if not supplied, will be set based on project_id)" + type = string + default = "" +} + +variable "user_managed_replication" { + type = list(object({ + location = string + kms_key_name = optional(string) + })) + description = "Replication parameters that will be used for defined secrets" + default = [] +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf new file mode 100644 index 0000000000..87a806a0ea --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.84" + } + random = { + source = "hashicorp/random" + version = ">= 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:htcondor-pool-secrets/v1.57.0" + } + + required_version = ">= 1.3.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md new file mode 100644 index 0000000000..5a403c0a38 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md @@ -0,0 +1,128 @@ +## Description + +This module creates the service accounts for use by the primary elements of an +[HTCondor pool][pool]: + +- Central Managers +- Access Points +- Execute Points + +Each service account is assigned common roles necessary for the VM to function +properly. In particular, nearly every VM requires the ability to read from Cloud +Storage buckets and write Cloud Logging entries. These roles are configurable +as described below. + +[pool]: https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-admin-manual.html#the-different-roles-a-machine-can-play + +### Example + +The following code snippet uses this module to create a startup script that +installs HTCondor software and configures an HTCondor Central Manager. A full +example can be found in the [examples README][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- + +```yaml +- id: network1 + source: modules/network/pre-existing-vpc + +- id: htcondor_install + source: community/modules/scripts/htcondor-install + +- id: htcondor_service_accounts + source: community/modules/scheduler/htcondor-service-accounts + +- id: htcondor_setup + source: community/modules/scheduler/htcondor-setup + use: + - network1 + - htcondor_service_accounts + +- id: htcondor_secrets + source: community/modules/scheduler/htcondor-pool-secrets + use: + - htcondor_service_accounts + +- id: htcondor_cm + source: community/modules/scheduler/htcondor-central-manager + use: + - network1 + - htcondor_secrets + - htcondor_service_accounts + - htcondor_setup + settings: + instance_image: + project: $(vars.project_id) + family: $(vars.new_image_family) + outputs: + - central_manager_name +``` + +## Support + +HTCondor is maintained by the [Center for High Throughput Computing][chtc] at +the University of Wisconsin-Madison. Support for HTCondor is available via: + +- [Discussion lists](https://htcondor.org/mail-lists/) +- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) +- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) + +[chtc]: https://chtc.cs.wisc.edu/ + +## License + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [access\_point\_service\_account](#module\_access\_point\_service\_account) | ../../../../community/modules/project/service-account | n/a | +| [central\_manager\_service\_account](#module\_central\_manager\_service\_account) | ../../../../community/modules/project/service-account | n/a | +| [execute\_point\_service\_account](#module\_execute\_point\_service\_account) | ../../../../community/modules/project/service-account | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_point\_roles](#input\_access\_point\_roles) | Project-wide roles for HTCondor Access Point service account | `list(string)` | 
[
  "compute.instanceAdmin.v1",
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | +| [central\_manager\_roles](#input\_central\_manager\_roles) | Project-wide roles for HTCondor Central Manager service account | `list(string)` | 
[
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [execute\_point\_roles](#input\_execute\_point\_roles) | Project-wide roles for HTCondor Execute Point service account | `list(string)` | 
[
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | +| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [access\_point\_service\_account\_email](#output\_access\_point\_service\_account\_email) | HTCondor Access Point Service Account (e-mail format) | +| [central\_manager\_service\_account\_email](#output\_central\_manager\_service\_account\_email) | HTCondor Central Manager Service Account (e-mail format) | +| [execute\_point\_service\_account\_email](#output\_execute\_point\_service\_account\_email) | HTCondor Execute Point Service Account (e-mail format) | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf new file mode 100644 index 0000000000..9d97b18642 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# NB: the community/modules/project/service-account module will not output the +# service account e-mail address until all IAM bindings have been created; if +# underlying implementation changes, this module should declare explicit +# depends_on the IAM bindings to prevent race conditions for services that +# require them + +module "access_point_service_account" { + source = "../../../../community/modules/project/service-account" + + project_id = var.project_id + display_name = "HTCondor Access Point" + deployment_name = var.deployment_name + name = "access" + project_roles = var.access_point_roles +} + +module "execute_point_service_account" { + source = "../../../../community/modules/project/service-account" + + project_id = var.project_id + display_name = "HTCondor Execute Point" + deployment_name = var.deployment_name + name = "execute" + project_roles = var.execute_point_roles +} + +module "central_manager_service_account" { + source = "../../../../community/modules/project/service-account" + + project_id = var.project_id + display_name = "HTCondor Central Manager" + deployment_name = var.deployment_name + name = "cm" + project_roles = var.central_manager_roles +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml new file mode 100644 index 0000000000..c4dcdffdf4 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - iam.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf new file mode 100644 index 0000000000..28f3a79457 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "access_point_service_account_email" { + description = "HTCondor Access Point Service Account (e-mail format)" + value = module.access_point_service_account.service_account_email +} + +output "central_manager_service_account_email" { + description = "HTCondor Central Manager Service Account (e-mail format)" + value = module.central_manager_service_account.service_account_email +} + +output "execute_point_service_account_email" { + description = "HTCondor Execute Point Service Account (e-mail format)" + value = module.execute_point_service_account.service_account_email +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf new file mode 100644 index 0000000000..ee186e0971 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf @@ -0,0 +1,56 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which HTCondor pool will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "access_point_roles" { + description = "Project-wide roles for HTCondor Access Point service account" + type = list(string) + default = [ + "compute.instanceAdmin.v1", + "monitoring.metricWriter", + "logging.logWriter", + "storage.objectViewer", + ] +} + +variable "central_manager_roles" { + description = "Project-wide roles for HTCondor Central Manager service account" + type = list(string) + default = [ + "monitoring.metricWriter", + "logging.logWriter", + "storage.objectViewer", + ] +} + +variable "execute_point_roles" { + description = "Project-wide roles for HTCondor Execute Point service account" + type = list(string) + default = [ + "monitoring.metricWriter", + "logging.logWriter", + "storage.objectViewer", + ] +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf new file mode 100644 index 0000000000..79b6fbde47 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/README.md new file mode 100644 index 0000000000..9c46f0a3c8 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/README.md @@ -0,0 +1,118 @@ +## Description + +This module creates a bucket in which to store HTCondor configurations and +a firewall rule that allows Managed Instance Group health checks to probe the +health of HTCondor VMs. + +### Example + +The following code snippet uses this module to create a startup script that +installs HTCondor software and configures an HTCondor Central Manager. A full +example can be found in the [examples README][htc-example]. + +[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- + +```yaml +- id: network1 + source: modules/network/pre-existing-vpc + +- id: htcondor_install + source: community/modules/scripts/htcondor-install + +- id: htcondor_service_accounts + source: community/modules/scheduler/htcondor-service-accounts + +- id: htcondor_setup + source: community/modules/scheduler/htcondor-setup + use: + - network1 + - htcondor_service_accounts + +- id: htcondor_secrets + source: community/modules/scheduler/htcondor-pool-secrets + use: + - htcondor_service_accounts + +- id: htcondor_cm + source: community/modules/scheduler/htcondor-central-manager + use: + - network1 + - htcondor_secrets + - htcondor_service_accounts + - htcondor_setup + settings: + instance_image: + project: $(vars.project_id) + family: $(vars.new_image_family) + outputs: + - central_manager_name +``` + +## Support + +HTCondor is maintained by the [Center for High Throughput Computing][chtc] at +the University of Wisconsin-Madison. Support for HTCondor is available via: + +- [Discussion lists](https://htcondor.org/mail-lists/) +- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) +- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) + +[chtc]: https://chtc.cs.wisc.edu/ + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [health\_check\_firewall\_rule](#module\_health\_check\_firewall\_rule) | ../../../../modules/network/firewall-rules | n/a | +| [htcondor\_bucket](#module\_htcondor\_bucket) | ../../../../community/modules/file-system/cloud-storage-bucket | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | Service account e-mail for HTCondor Access Point | `string` | n/a | yes | +| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | Service account e-mail for HTCondor Central Manager | `string` | n/a | yes | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | Service account e-mail for HTCondor Execute Points | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | +| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which Central Managers will be placed. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [htcondor\_bucket\_name](#output\_htcondor\_bucket\_name) | Name of the HTCondor configuration bucket | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf new file mode 100644 index 0000000000..ad471fddad --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf @@ -0,0 +1,68 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "htcondor-setup", ghpc_role = "scheduler" }) +} + +locals { + service_account_iam_email = [ + "serviceAccount:${var.access_point_service_account_email}", + "serviceAccount:${var.central_manager_service_account_email}", + "serviceAccount:${var.execute_point_service_account_email}", + ] + service_account_email = [ + var.access_point_service_account_email, + var.central_manager_service_account_email, + var.execute_point_service_account_email, + ] +} + +module "health_check_firewall_rule" { + source = "../../../../modules/network/firewall-rules" + + subnetwork_self_link = var.subnetwork_self_link + + ingress_rules = [{ + name = "allow-health-check-${var.deployment_name}" + description = "Allow Managed Instance Group Health Checks for HTCondor VMs" + direction = "INGRESS" + source_ranges = [ + "130.211.0.0/22", + "35.191.0.0/16", + ] + target_service_accounts = local.service_account_email + allow = [{ + protocol = "tcp" + ports = ["9618"] + }] + }] +} + +module "htcondor_bucket" { + source = "../../../../community/modules/file-system/cloud-storage-bucket" + + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + name_prefix = "${var.deployment_name}-htcondor-config" + random_suffix = true + labels = local.labels + viewers = local.service_account_iam_email + + use_deployment_name_in_bucket_name = false +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml new file mode 100644 index 0000000000..7b4918b962 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - iam.googleapis.com + - storage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf new file mode 100644 index 0000000000..a44223faee --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "htcondor_bucket_name" { + description = "Name of the HTCondor configuration bucket" + value = module.htcondor_bucket.gcs_bucket_name + + # ensure that all IAM bindings to the bucket and firewall rules are active + # before this modules output is allowed to propagate + depends_on = [ + module.htcondor_bucket, + module.health_check_firewall_rule + ] +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf new file mode 100644 index 0000000000..147a2ca88d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which HTCondor pool will be created" + type = string +} + +variable "deployment_name" { + description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." + type = string +} + +variable "labels" { + description = "Labels to add to resources. List key, value pairs." + type = map(string) +} + +variable "region" { + description = "Default region for creating resources" + type = string +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork in which Central Managers will be placed." + type = string +} + +variable "access_point_service_account_email" { + description = "Service account e-mail for HTCondor Access Point" + type = string +} + +variable "central_manager_service_account_email" { + description = "Service account e-mail for HTCondor Central Manager" + type = string +} + +variable "execute_point_service_account_email" { + description = "Service account e-mail for HTCondor Execute Points" + type = string +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf new file mode 100644 index 0000000000..79b6fbde47 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md new file mode 100644 index 0000000000..7ddb6058c4 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md @@ -0,0 +1,405 @@ +## Description + +This module creates a slurm controller node via the internal +[slurm\_instance\_template] module. + +More information about Slurm On GCP can be found at the +[project's GitHub page][slurm-gcp] and in the +[Slurm on Google Cloud User Guide][slurm-ug]. + +The [user guide][slurm-ug] provides detailed instructions on customizing and +enhancing the Slurm on GCP cluster as well as recommendations on configuring the +controller for optimal performance at different scales. + +[slurm\_instance\_template]: /community/modules/internal/slurm-gcp/instance_template/README.md +[slurm-ug]: https://goo.gle/slurm-gcp-user-guide. +[enable\_cleanup\_compute]: #input\_enable\_cleanup\_compute +[enable\_cleanup\_subscriptions]: #input\_enable\_cleanup\_subscriptions +[enable\_reconfigure]: #input\_enable\_reconfigure + +### Example + +```yaml +- id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + use: + - network + - homefs + - compute_partition + settings: + machine_type: c2-standard-8 +``` + +This creates a controller node with the following attributes: + +* connected to the primary subnetwork of `network` +* the filesystem with the ID `homefs` (defined elsewhere in the blueprint) + mounted +* One partition with the ID `compute_partition` (defined elsewhere in the + blueprint) +* machine type upgraded from the default `c2-standard-4` to `c2-standard-8` + +### Live Cluster Reconfiguration + +The `schedmd-slurm-gcp-v6-controller` module supports the reconfiguration of +partitions and slurm configuration in a running, active cluster. + +To reconfigure a running cluster: + +1. Edit the blueprint with the desired configuration changes +2. Call `gcluster create -w` to overwrite the deployment directory +3. Follow instructions in terminal to deploy + +The following are examples of updates that can be made to a running cluster: + +* Add or remove a partition to the cluster +* Resize an existing partition +* Attach new network storage to an existing partition + +> **NOTE**: Changing the VM `machine_type` of a partition may not work. +> It is better to create a new partition and delete the old one. + +## Custom Images + +For more information on creating valid custom images for the controller VM +instance or for custom instance templates, see our [vm-images.md] documentation +page. + +[vm-images.md]: ../../../../docs/vm-images.md#slurm-on-gcp-custom-images + +## GPU Support + +More information on GPU support in Slurm on GCP and other Cluster Toolkit modules +can be found at [docs/gpu-support.md](../../../../docs/gpu-support.md) + +## Reservation for Scheduled Maintenance + +A [maintenance event](https://cloud.google.com/compute/docs/instances/host-maintenance-overview#maintenanceevents) is when a compute engine stops a VM to perform a hardware or +software update which is determined by the host maintenance policy. This can +also affect the running jobs if the maintenance kicks in. Now, Customers can +protect jobs from getting terminated due to maintenance using the cluster +toolkit. You can enable creation of reservation for scheduled maintenance for +your compute nodeset and Slurm will reserve your node for maintenance during the +maintenance window. If you try to schedule any jobs which overlap with the +maintenance reservation, Slurm would not schedule any job. + +You can specify in your blueprint like + +```yaml + - id: compute_nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [network] + settings: + enable_maintenance_reservation: true +``` + +To enable creation of reservation for maintenance. + +While running job on slurm cluster, you can specify total run time of the job +using [-t flag](https://slurm.schedmd.com/srun.html#OPT_time).This would only +run the job outside of the maintenance window. + +```shell +srun -n1 -pcompute -t 10:00 +``` + +Currently upcoming maintenance notification is supported in ALPHA version of +compute API. You can update the API version from your blueprint, + +```yaml + - id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + settings: + endpoint_versions: + compute: "alpha" +``` + +## Opportunistic GCP maintenance in Slurm + +Customers can also enable running GCP maintenance as Slurm job opportunistically +to perform early maintenance. If a node is detected for maintenance, Slurm will +create a job to perform maintenance and put it in the job queue. + +If [backfill](https://slurm.schedmd.com/sched_config.html#backfill) scheduler is +used, Slurm will backfill maintenance job if it can find any empty time window. + +Customer can also choose builtin scheduler type. In this case, Slurm would run +maintenance job in strictly priority order. If the maintenance job doesn't kick +in, then forced maintenance will take place at scheduled window. + +Customer can enable this feature at nodeset level by, + +```yaml + - id: debug_nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [network] + settings: + enable_opportunistic_maintenance: true +``` + +## Placement Max Distance + +When using +[enable_placement](../../../../community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md#input_enable_placement) +with Slurm, Google Compute Engine will attempt to place VMs as physically close +together as possible. Capacity constraints at the time of VM creation may still +force VMs to be spread across multiple racks. Google provides the `max-distance` +flag which can used to control the maximum spreading allowed. Read more about +`max-distance` in the +[official docs](https://cloud.google.com/compute/docs/instances/use-compact-placement-policies +). + +You can use the `placement_max_distance` setting on the nodeset module to control the `max-distance` behavior. See the following example: + +```yaml + - id: nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [ network ] + settings: + machine_type: c2-standard-4 + node_count_dynamic_max: 30 + enable_placement: true + placement_max_distance: 1 + +> [!NOTE] +> `schedmd-slurm-gcp-v6-nodeset.settings.enable_placement: true` must also be +> set for placement_max_distance to take effect. + +In the above case using a value of 1 will restrict VM to be placed on the same +rack. You can confirm that the `max-distance` was applied by calling the +following command while jobs are running: + +```shell +gcloud beta compute resource-policies list \ + --format='yaml(name,groupPlacementPolicy.maxDistance)' +``` + +> [!WARNING] +> If a zone lacks capacity, using a lower `max-distance` value (such as 1) is +> more likely to cause VMs creation to fail. + +## TreeWidth and Node Communication + +Slurm uses a fan out mechanism to communicate large groups of nodes. The shape +of this fan out tree is determined by the +[TreeWidth](https://slurm.schedmd.com/slurm.conf.html#OPT_TreeWidth) +configuration variable. + +In the cloud, this fan out mechanism can become unstable when nodes restart with +new IP addresses. You can enforce that all nodes communicate directly with the +controller by setting TreeWidth to a value >= largest partition. + +If the largest partition was 200 nodes, configure the blueprint as follows: + +```yaml + - id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + ... + settings: + cloud_parameters: + tree_width: 200 +``` + +The default has been set to 128. Values above this have not been fully tested +and may cause congestion on the controller. A more scalable solution is under +way. + +## ResumeRate and Node Resumption + +The `ResumeRate` parameter in `slurm.conf` controls the maximum number of nodes +that Slurm attempts to resume (power up) per minute. This is particularly +important in cloud environments where auto-scaling can lead to a large number of +nodes starting concurrently. + +When many nodes start simultaneously, they can place a heavy load on shared +resources, especially shared filesystems, as they all try to mount filesystems +and access configuration files at the same time. By limiting the `ResumeRate`, +you can stagger the node startup process, reducing the peak load on these shared +resources and improving overall cluster stability during scaling events. + +For example, to limit the node resumption rate to 100 nodes per minute, +configure the blueprint as follows: + +```yaml + - id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + ... + settings: + cloud_parameters: + resume_rate: 100 +``` + +Adjust this value based on the capabilities of your shared filesystem and the +expected scaling behavior of your cluster. + +## Support +The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform +modules. For support with the underlying modules, see the instructions in the +[slurm-gcp README][slurm-gcp-readme]. + +[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp +[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 4.84 | +| [google-beta](#requirement\_google-beta) | >= 6.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.84 | +| [google-beta](#provider\_google-beta) | >= 6.0.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [bucket](#module\_bucket) | terraform-google-modules/cloud-storage/google | ~> 6.1 | +| [daos\_network\_storage\_scripts](#module\_daos\_network\_storage\_scripts) | ../../../../modules/scripts/startup-script | n/a | +| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | +| [login](#module\_login) | ../../internal/slurm-gcp/login | n/a | +| [nodeset\_cleanup](#module\_nodeset\_cleanup) | ./modules/cleanup_compute | n/a | +| [nodeset\_cleanup\_tpu](#module\_nodeset\_cleanup\_tpu) | ./modules/cleanup_tpu | n/a | +| [slurm\_controller\_template](#module\_slurm\_controller\_template) | ../../internal/slurm-gcp/instance_template | n/a | +| [slurm\_files](#module\_slurm\_files) | ./modules/slurm_files | n/a | +| [slurm\_nodeset\_template](#module\_slurm\_nodeset\_template) | ../../internal/slurm-gcp/instance_template | n/a | +| [slurm\_nodeset\_tpu](#module\_slurm\_nodeset\_tpu) | ../../internal/slurm-gcp/nodeset_tpu | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_instance_from_template.controller](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance_from_template) | resource | +| [google_compute_disk.controller_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [google_secret_manager_secret.cloudsql](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | +| [google_secret_manager_secret_iam_member.cloudsql_secret_accessor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | +| [google_secret_manager_secret_version.cloudsql_version](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | +| [google_storage_bucket_iam_member.legacy_readers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_member) | resource | +| [google_storage_bucket_iam_member.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_member) | resource | +| [google_storage_bucket_object.parition_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_compute_image.slurm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | +| [google_project.controller_project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    disk_name                  = string
    device_name                = string
    disk_type                  = string
    disk_size_gb               = number
    disk_labels                = map(string)
    auto_delete                = bool
    boot                       = bool
    disk_resource_manager_tags = map(string)
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | +| [bucket\_dir](#input\_bucket\_dir) | Bucket directory for cluster files to be put into. If not specified, then one will be chosen based on slurm\_cluster\_name. | `string` | `null` | no | +| [bucket\_name](#input\_bucket\_name) | Name of GCS bucket.
Ignored when 'create\_bucket' is true. | `string` | `null` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | +| [cgroup\_conf\_tpl](#input\_cgroup\_conf\_tpl) | Slurm cgroup.conf template file path. | `string` | `null` | no | +| [cloud\_parameters](#input\_cloud\_parameters) | cloud.conf options. Defaults inherited from [Slurm GCP repo](https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/terraform/slurm_cluster/modules/slurm_files/README_TF.md#input_cloud_parameters) | 
object({
    no_comma_params      = optional(bool, false)
    private_data         = optional(list(string))
    scheduler_parameters = optional(list(string))
    resume_rate          = optional(number)
    resume_timeout       = optional(number)
    suspend_rate         = optional(number)
    suspend_timeout      = optional(number)
    topology_plugin      = optional(string)
    topology_param       = optional(string)
    tree_width           = optional(number)
  })
| `{}` | no | +| [cloudsql](#input\_cloudsql) | Use this database instead of the one on the controller.
server\_ip : Address of the database server.
user : The user to access the database as.
password : The password, given the user, to access the given database. (sensitive)
db\_name : The database to access.
user\_managed\_replication : The list of location and (optional) kms\_key\_name for secret | 
object({
    server_ip = string
    user      = string
    password  = string # sensitive
    db_name   = string
    user_managed_replication = optional(list(object({
      location     = string
      kms_key_name = optional(string)
    })), [])
  })
| `null` | no | +| [compute\_startup\_script](#input\_compute\_startup\_script) | DEPRECATED: `compute_startup_script` has been deprecated.
Use `startup_script` of nodeset module instead. | `any` | `null` | no | +| [compute\_startup\_scripts\_timeout](#input\_compute\_startup\_scripts\_timeout) | The timeout (seconds) applied to each startup script in compute nodes. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [controller\_network\_attachment](#input\_controller\_network\_attachment) | SelfLink for NetworkAttachment to be attached to the controller, if any. | `string` | `null` | no | +| [controller\_project\_id](#input\_controller\_project\_id) | Optionally. Provision controller and config bucket in the different project | `string` | `null` | no | +| [controller\_startup\_script](#input\_controller\_startup\_script) | Startup script used by the controller VM. | `string` | `"# no-op"` | no | +| [controller\_startup\_scripts\_timeout](#input\_controller\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in controller\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [controller\_state\_disk](#input\_controller\_state\_disk) | A disk that will be attached to the controller instance template to save state of slurm. The disk is created and used by default.
To disable this feature, set this variable to null.

NOTE: This will not save the contents at /opt/apps and /home. To preserve those, they must be saved externally. | 
object({
    type = string
    size = number
  })
| 
{
  "size": 50,
  "type": "pd-ssd"
}
| no | +| [create\_bucket](#input\_create\_bucket) | Create GCS bucket instead of using an existing one. | `bool` | `true` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment. | `string` | n/a | yes | +| [disable\_controller\_public\_ips](#input\_disable\_controller\_public\_ips) | DEPRECATED: Use `enable_controller_public_ips` instead. | `bool` | `null` | no | +| [disable\_default\_mounts](#input\_disable\_default\_mounts) | DEPRECATED: Use `enable_default_mounts` instead. | `bool` | `null` | no | +| [disable\_smt](#input\_disable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | +| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | +| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `50` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-ssd"` | no | +| [enable\_bigquery\_load](#input\_enable\_bigquery\_load) | Enables loading of cluster job usage into big query.

NOTE: Requires Google Bigquery API. | `bool` | `false` | no | +| [enable\_chs\_gpu\_health\_check\_epilog](#input\_enable\_chs\_gpu\_health\_check\_epilog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as an epilog script after completing a job step from a new job allocation.
Compute nodes that fail GPU health check during epilog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | +| [enable\_chs\_gpu\_health\_check\_prolog](#input\_enable\_chs\_gpu\_health\_check\_prolog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as a prolog script whenever it is asked to run a job step from a new job allocation. Compute nodes that fail GPU health check during prolog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | +| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of compute nodes and resource policies (e.g.
placement groups) managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed compute nodes will be destroyed. | `bool` | `true` | no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_controller\_public\_ips](#input\_enable\_controller\_public\_ips) | If set to true. The controller will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | +| [enable\_debug\_logging](#input\_enable\_debug\_logging) | Enables debug logging mode. | `bool` | `false` | no | +| [enable\_default\_mounts](#input\_enable\_default\_mounts) | Enable default global network storage from the controller
- /home
- /apps | `bool` | `true` | no | +| [enable\_devel](#input\_enable\_devel) | DEPRECATED: `enable_devel` is always on. | `bool` | `null` | no | +| [enable\_external\_prolog\_epilog](#input\_enable\_external\_prolog\_epilog) | Automatically enable a script that will execute prolog and epilog scripts
shared by NFS from the controller to compute nodes. Find more details at:
https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/tools/prologs-epilogs/README.md | `bool` | `null` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_slurm\_auth](#input\_enable\_slurm\_auth) | Enables slurm authentication instead of munge. | `bool` | `false` | no | +| [enable\_slurm\_gcp\_plugins](#input\_enable\_slurm\_gcp\_plugins) | DEPRECATED: Slurm GCP plugins have been deprecated.
Instead of 'max\_hops' plugin please use the 'placement\_max\_distance' nodeset property.
Instead of 'enable\_vpmu' plugin please use 'advanced\_machine\_features.performance\_monitoring\_unit' nodeset property. | `any` | `null` | no | +| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| 
{
  "compute": "beta"
}
| no | +| [epilog\_scripts](#input\_epilog\_scripts) | List of scripts to be used for Epilog. Programs for the slurmd to execute
on every node when a user's job completes.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Epilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [extra\_logging\_flags](#input\_extra\_logging\_flags) | The only available flag is `trace_api` | `map(bool)` | `{}` | no | +| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | `""` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm controller VM instance.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-9-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | +| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | +| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for controller. | `string` | `null` | no | +| [labels](#input\_labels) | Labels, provided as a map. | `map(string)` | `{}` | no | +| [login\_network\_storage](#input\_login\_network\_storage) | An array of network attached storage mounts to be configured on all login nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | +| [login\_nodes](#input\_login\_nodes) | List of slurm login instance definitions. | 
list(object({
    group_name = string
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    additional_networks = optional(list(object({
      access_config = optional(list(object({
        nat_ip       = string
        network_tier = string
      })), [])
      alias_ip_range = optional(list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      })), [])
      ipv6_access_config = optional(list(object({
        network_tier = string
      })), [])
      network            = optional(string)
      network_ip         = optional(string, "")
      nic_type           = optional(string)
      queue_count        = optional(number)
      stack_type         = optional(string)
      subnetwork         = optional(string)
      subnetwork_project = optional(string)
    })), [])
    bandwidth_tier             = optional(string, "platform_default")
    can_ip_forward             = optional(bool, false)
    disk_auto_delete           = optional(bool, true)
    disk_labels                = optional(map(string), {})
    disk_resource_manager_tags = optional(map(string), {})
    disk_size_gb               = optional(number)
    disk_type                  = optional(string, "n1-standard-1")
    enable_confidential_vm     = optional(bool, false)
    enable_oslogin             = optional(bool, true)
    enable_shielded_vm         = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    metadata              = optional(map(string), {})
    min_cpu_platform      = optional(string)
    num_instances         = optional(number, 1)
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    static_ips           = optional(list(string), [])
    subnetwork           = string
    spot                 = optional(bool, false)
    tags                 = optional(list(string), [])
    zone                 = optional(string)
    termination_action   = optional(string)
  }))
| `[]` | no | +| [login\_startup\_script](#input\_login\_startup\_script) | Startup script used by the login VMs. | `string` | `"# no-op"` | no | +| [login\_startup\_scripts\_timeout](#input\_login\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in login\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"c2-standard-4"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on all instances. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = optional(map(string))
    mount_runner          = optional(map(string))
  }))
| `[]` | no | +| [nodeset](#input\_nodeset) | Define nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 1)
    node_conf              = optional(map(string), {})
    nodeset_name           = string
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    bandwidth_tier                   = optional(string, "platform_default")
    can_ip_forward                   = optional(bool, false)
    disk_auto_delete                 = optional(bool, true)
    disk_labels                      = optional(map(string), {})
    disk_resource_manager_tags       = optional(map(string), {})
    disk_size_gb                     = optional(number)
    disk_type                        = optional(string)
    enable_confidential_vm           = optional(bool, false)
    enable_placement                 = optional(bool, false)
    placement_max_distance           = optional(number, null)
    enable_oslogin                   = optional(bool, true)
    enable_shielded_vm               = optional(bool, false)
    enable_maintenance_reservation   = optional(bool, false)
    enable_opportunistic_maintenance = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    dws_flex = object({
      enabled          = bool
      max_run_duration = number
      use_job_duration = bool
      use_bulk_insert  = bool
    })
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    maintenance_interval     = optional(string)
    instance_properties_json = string
    metadata                 = optional(map(string), {})
    min_cpu_platform         = optional(string)
    network_tier             = optional(string, "STANDARD")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    subnetwork_self_link = string
    additional_networks = optional(list(object({
      network            = string
      subnetwork         = string
      subnetwork_project = string
      network_ip         = string
      nic_type           = string
      stack_type         = string
      queue_count        = number
      access_config = list(object({
        nat_ip       = string
        network_tier = string
      }))
      ipv6_access_config = list(object({
        network_tier = string
      }))
      alias_ip_range = list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      }))
    })))
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    spot               = optional(bool, false)
    tags               = optional(list(string), [])
    termination_action = optional(string)
    reservation_name   = optional(string)
    future_reservation = string
    startup_script = optional(list(object({
      filename = string
    content = string })), [])

    zone_target_shape = string
    zone_policy_allow = set(string)
    zone_policy_deny  = set(string)
  }))
| `[]` | no | +| [nodeset\_dyn](#input\_nodeset\_dyn) | Defines dynamic nodesets, as a list. | 
list(object({
    nodeset_name    = string
    nodeset_feature = string
  }))
| `[]` | no | +| [nodeset\_tpu](#input\_nodeset\_tpu) | Define TPU nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 5)
    nodeset_name           = string
    enable_public_ip       = optional(bool, false)
    node_type              = string
    accelerator_config = optional(object({
      topology = string
      version  = string
      }), {
      topology = ""
      version  = ""
    })
    tf_version   = string
    preemptible  = optional(bool, false)
    preserve_tpu = optional(bool, false)
    zone         = string
    data_disks   = optional(list(string), [])
    docker_image = optional(string, "")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    subnetwork = string
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    project_id = string
    reserved   = optional(string, false)
  }))
| `[]` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy. | `string` | `"MIGRATE"` | no | +| [partitions](#input\_partitions) | Cluster partitions as a list. See module slurm\_partition. | 
list(object({
    partition_name        = string
    partition_conf        = optional(map(string), {})
    partition_nodeset     = optional(list(string), [])
    partition_nodeset_dyn = optional(list(string), [])
    partition_nodeset_tpu = optional(list(string), [])
    enable_job_exclusive  = optional(bool, false)
  }))
| `[]` | no | +| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [prolog\_scripts](#input\_prolog\_scripts) | List of scripts to be used for Prolog. Programs for the slurmd to execute
whenever it is asked to run a job step from a new job allocation.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Prolog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [region](#input\_region) | The default region to place resources in. | `string` | n/a | yes | +| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the controller instance. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the controller instance. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used for resource naming and slurm accounting.
If not provided it will default to the first 8 characters of the deployment name (removing any invalid characters). | `string` | `null` | no | +| [slurm\_conf\_template](#input\_slurm\_conf\_template) | Slurm slurm.conf template. Content of the file in 'slurm\_conf\_tpl' is used if this is not set. | `string` | `null` | no | +| [slurm\_conf\_tpl](#input\_slurm\_conf\_tpl) | Slurm slurm.conf template file path. This path is used only if raw content is not provided in 'slurm\_conf\_template'. | `string` | `null` | no | +| [slurmdbd\_conf\_tpl](#input\_slurmdbd\_conf\_tpl) | Slurm slurmdbd.conf template file path. | `string` | `null` | no | +| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances. | `list(string)` | `[]` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | +| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | +| [task\_epilog\_scripts](#input\_task\_epilog\_scripts) | List of scripts to be used for TaskEpilog. Programs for the slurmd to execute
as the slurm job's owner after termination of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskEpilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [task\_prolog\_scripts](#input\_task\_prolog\_scripts) | List of scripts to be used for TaskProlog. Programs for the slurmd to execute
as the slurm job's owner prior to initiation of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskProlog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | +| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be
spread across available zones in the region. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Post deployment instructions. | +| [slurm\_bucket\_dir](#output\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | +| [slurm\_bucket\_name](#output\_slurm\_bucket\_name) | GCS Bucket name of Slurm cluster file storage. | +| [slurm\_bucket\_path](#output\_slurm\_bucket\_path) | Bucket path used by cluster. | +| [slurm\_cluster\_name](#output\_slurm\_cluster\_name) | Slurm cluster name. | +| [slurm\_controller\_instance](#output\_slurm\_controller\_instance) | Compute instance of controller node | +| [slurm\_login\_instances](#output\_slurm\_login\_instances) | Compute instances of login nodes | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf new file mode 100644 index 0000000000..4a887b99cf --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf @@ -0,0 +1,213 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module "gpu" { + source = "../../../../modules/internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + additional_disks = [ + for ad in var.additional_disks : { + disk_name = ad.disk_name + device_name = ad.device_name + disk_type = ad.disk_type + disk_size_gb = ad.disk_size_gb + disk_labels = merge(ad.disk_labels, local.labels) + auto_delete = ad.auto_delete + boot = ad.boot + disk_resource_manager_tags = ad.disk_resource_manager_tags + } + ] + + state_disk = var.controller_state_disk != null ? [{ + source = google_compute_disk.controller_disk[0].name + device_name = google_compute_disk.controller_disk[0].name + disk_labels = null + auto_delete = false + boot = false + }] : [] + + synth_def_sa_email = "${data.google_project.controller_project.number}-compute@developer.gserviceaccount.com" + + service_account = { + email = coalesce(var.service_account_email, local.synth_def_sa_email) + scopes = var.service_account_scopes + } + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + metadata = merge( + local.disable_automatic_updates_metadata, + var.metadata, + local.universe_domain + ) + + controller_project_id = coalesce(var.controller_project_id, var.project_id) +} + +data "google_project" "controller_project" { + project_id = local.controller_project_id +} + +resource "google_compute_disk" "controller_disk" { + count = var.controller_state_disk != null ? 1 : 0 + + project = local.controller_project_id + name = "${local.slurm_cluster_name}-controller-save" + type = var.controller_state_disk.type + size = var.controller_state_disk.size + zone = var.zone +} + +# INSTANCE TEMPLATE +module "slurm_controller_template" { + source = "../../internal/slurm-gcp/instance_template" + + project_id = local.controller_project_id + region = var.region + slurm_instance_role = "controller" + slurm_cluster_name = local.slurm_cluster_name + labels = local.labels + + disk_auto_delete = var.disk_auto_delete + disk_labels = merge(var.disk_labels, local.labels) + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + disk_resource_manager_tags = var.disk_resource_manager_tags + additional_disks = concat(local.additional_disks, local.state_disk) + + bandwidth_tier = var.bandwidth_tier + slurm_bucket_path = module.slurm_files.slurm_bucket_path + can_ip_forward = var.can_ip_forward + advanced_machine_features = var.advanced_machine_features + resource_manager_tags = var.resource_manager_tags + + enable_confidential_vm = var.enable_confidential_vm + enable_oslogin = var.enable_oslogin + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config + + gpu = one(module.gpu.guest_accelerator) + + machine_type = var.machine_type + metadata = local.metadata + min_cpu_platform = var.min_cpu_platform + + on_host_maintenance = var.on_host_maintenance + preemptible = var.preemptible + service_account = local.service_account + + source_image_family = local.source_image_family # requires source_image_logic.tf + source_image_project = local.source_image_project_normalized # requires source_image_logic.tf + source_image = local.source_image # requires source_image_logic.tf + + subnetwork = var.subnetwork_self_link + + tags = concat([local.slurm_cluster_name], var.tags) + # termination_action = TODO: add support for termination_action (?) +} + +# INSTANCE +resource "google_compute_instance_from_template" "controller" { + provider = google-beta + + name = "${local.slurm_cluster_name}-controller" + project = local.controller_project_id + zone = var.zone + source_instance_template = module.slurm_controller_template.self_link + # Due to https://github.com/hashicorp/terraform-provider-google/issues/21693 + # we have to explicitly override instance labels instead of inheriting them from template. + labels = module.slurm_controller_template.labels + + allow_stopping_for_update = true + + # Can't rely on template to specify nics due to usage of static_ip + network_interface { + dynamic "access_config" { + for_each = var.enable_controller_public_ips ? ["unit"] : [] + content { + nat_ip = null + network_tier = null + } + } + network_ip = length(var.static_ips) == 0 ? "" : var.static_ips[0] + subnetwork = var.subnetwork_self_link + } + + dynamic "network_interface" { + for_each = var.controller_network_attachment != null ? [1] : [] + content { + network_attachment = var.controller_network_attachment + } + } +} + +moved { + from = module.slurm_controller_instance.google_compute_instance_from_template.slurm_instance[0] + to = google_compute_instance_from_template.controller +} + +# SECRETS: CLOUDSQL +resource "google_secret_manager_secret" "cloudsql" { + count = var.cloudsql != null ? 1 : 0 + + secret_id = "${local.slurm_cluster_name}-slurm-secret-cloudsql" + project = var.project_id + + replication { + dynamic "auto" { + for_each = length(var.cloudsql.user_managed_replication) == 0 ? [1] : [] + content {} + } + dynamic "user_managed" { + for_each = length(var.cloudsql.user_managed_replication) == 0 ? [] : [1] + content { + dynamic "replicas" { + for_each = nonsensitive(var.cloudsql.user_managed_replication) + content { + location = replicas.value.location + dynamic "customer_managed_encryption" { + for_each = compact([replicas.value.kms_key_name]) + content { + kms_key_name = customer_managed_encryption.value + } + } + } + } + } + } + } + + labels = { + slurm_cluster_name = local.slurm_cluster_name + } +} + +resource "google_secret_manager_secret_version" "cloudsql_version" { + count = var.cloudsql != null ? 1 : 0 + + secret = google_secret_manager_secret.cloudsql[0].id + secret_data = jsonencode(var.cloudsql) +} + +resource "google_secret_manager_secret_iam_member" "cloudsql_secret_accessor" { + count = var.cloudsql != null ? 1 : 0 + + secret_id = google_secret_manager_secret.cloudsql[0].id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${local.service_account.email}" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl new file mode 100644 index 0000000000..219bdc5227 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl @@ -0,0 +1,65 @@ +# slurm.conf +# https://slurm.schedmd.com/high_throughput.html + +ProctrackType=proctrack/cgroup +SlurmctldPidFile=/var/run/slurm/slurmctld.pid +SlurmdPidFile=/var/run/slurm/slurmd.pid +TaskPlugin=task/affinity,task/cgroup +MaxArraySize=10001 +MaxJobCount=500000 +MaxNodeCount=65536 +MinJobAge=60 + +# +# +# SCHEDULING +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core_Memory + +# +# +# LOGGING AND ACCOUNTING +SlurmctldDebug=error +SlurmdDebug=error + +# +# +# TIMERS +MessageTimeout=60 + +################################################################################ +# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # +################################################################################ + +SlurmctldHost={control_host}({control_addr}) + +AuthType=auth/{auth_key} +AuthInfo=cred_expire=120 +AuthAltTypes=auth/jwt +CredType=cred/{auth_key} +MpiDefault={mpi_default} +ReturnToService=2 +SlurmctldPort={control_host_port} +SlurmdPort=6818 +SlurmdSpoolDir=/var/spool/slurmd +SlurmUser=slurm +StateSaveLocation={state_save} + +# +# +# LOGGING AND ACCOUNTING +AccountingStorageType=accounting_storage/slurmdbd +AccountingStorageHost={accounting_storage_host} +ClusterName={name} +SlurmctldLogFile={slurmlog}/slurmctld.log +SlurmdLogFile={slurmlog}/slurmd-%n.log + +# +# +# GENERATED CLOUD CONFIGURATIONS +include cloud.conf + +################################################################################ +# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # +################################################################################ diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl new file mode 100644 index 0000000000..93ac47e341 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl @@ -0,0 +1,34 @@ +# slurmdbd.conf +# https://slurm.schedmd.com/slurmdbd.conf.html + +DebugLevel=info +PidFile=/var/run/slurm/slurmdbd.pid + +# https://slurm.schedmd.com/slurmdbd.conf.html#OPT_CommitDelay +CommitDelay=1 + +################################################################################ +# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # +################################################################################ + +AuthType=auth/{auth_key} +AuthAltTypes=auth/jwt +AuthAltParameters=jwt_key={state_save}/jwt_hs256.key + +DbdHost={control_host} + +LogFile={slurmlog}/slurmdbd.log + +SlurmUser=slurm + +StorageLoc={db_name} + +StorageType=accounting_storage/mysql +StorageHost={db_host} +StoragePort={db_port} +StorageUser={db_user} +StoragePass={db_pass} + +################################################################################ +# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # +################################################################################ diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl new file mode 100644 index 0000000000..d3f2615a68 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl @@ -0,0 +1,71 @@ +# slurm.conf +# https://slurm.schedmd.com/slurm.conf.html +# https://slurm.schedmd.com/configurator.html + +ProctrackType=proctrack/cgroup +SlurmctldPidFile=/var/run/slurm/slurmctld.pid +SlurmdPidFile=/var/run/slurm/slurmd.pid +TaskPlugin=task/affinity,task/cgroup +MaxNodeCount=64000 + +# +# +# SCHEDULING +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core_Memory + +# +# +# LOGGING AND ACCOUNTING +AccountingStoreFlags=job_comment +JobAcctGatherFrequency=30 +JobAcctGatherType=jobacct_gather/cgroup +SlurmctldDebug=info +SlurmdDebug=info +DebugFlags=Power + +# +# +# TIMERS +MessageTimeout=600 +BatchStartTimeout=600 +PrologEpilogTimeout=600 +PrologFlags=Contain + +################################################################################ +# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # +################################################################################ + +SlurmctldHost={control_host}({control_addr}) + + +AuthType=auth/{auth_key} +AuthInfo=cred_expire=600 +AuthAltTypes=auth/jwt +CredType=cred/{auth_key} +MpiDefault={mpi_default} +ReturnToService=2 +SlurmctldPort={control_host_port} +SlurmdPort=6818 +SlurmdSpoolDir=/var/spool/slurmd +SlurmUser=slurm +StateSaveLocation={state_save} + +# +# +# LOGGING AND ACCOUNTING +AccountingStorageType=accounting_storage/slurmdbd +AccountingStorageHost={accounting_storage_host} +ClusterName={name} +SlurmctldLogFile={slurmlog}/slurmctld.log +SlurmdLogFile={slurmlog}/slurmd-%n.log + +# +# +# GENERATED CLOUD CONFIGURATIONS +include cloud.conf + +################################################################################ +# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # +################################################################################ diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf new file mode 100644 index 0000000000..21e915a125 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf @@ -0,0 +1,50 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +locals { + # TODO: deprecate `var.login_[ startup_script, startup_scripts_timeout, network_storage]` + # in favour of vars defined in user-facing login module + ghpc_startup_login = [{ + filename = "ghpc_startup.sh" + content = var.login_startup_script + }] + + login_startup_scripts = concat(local.common_scripts, local.ghpc_startup_login) +} + +module "login" { + source = "../../internal/slurm-gcp/login" + for_each = { for x in var.login_nodes : x.group_name => x } + + project_id = var.project_id + + slurm_cluster_name = local.slurm_cluster_name + slurm_bucket_path = module.slurm_files.slurm_bucket_path + slurm_bucket_name = module.slurm_files.bucket_name + slurm_bucket_dir = module.slurm_files.bucket_dir + + login_nodes = each.value + + startup_scripts = local.login_startup_scripts + startup_scripts_timeout = var.login_startup_scripts_timeout + + network_storage = var.login_network_storage + + universe_domain = var.universe_domain + + # trigger replacement of login nodes when the controller instance is replaced + # Needed for re-mounting volumes hosted on controller + replace_trigger = google_compute_instance_from_template.controller.self_link +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf new file mode 100644 index 0000000000..7622bdffef --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-controller", ghpc_role = "scheduler" }) +} + +locals { + # Since deployment name may be used to create a cluster name, we remove any invalid character from the beginning + # Also, slurm imposed a lot of restrictions to this name, so we format it to an acceptable string + tmp_cluster_name = substr(replace(lower(var.deployment_name), "/^[^a-z]*|[^a-z0-9]/", ""), 0, 10) + slurm_cluster_name = coalesce(var.slurm_cluster_name, local.tmp_cluster_name) + + universe_domain = { "universe_domain" = var.universe_domain } +} + +# See +# * slurm_files.tf +# * controller.tf +# * partition.tf +# * login.tf diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml new file mode 100644 index 0000000000..7b4918b962 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - iam.googleapis.com + - storage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md new file mode 100644 index 0000000000..002bf14145 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md @@ -0,0 +1,42 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [null](#provider\_null) | >= 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.dependencies](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of compute nodes and resource policies (e.g.
placement groups) managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed compute nodes will be destroyed. | `bool` | n/a | yes | +| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | +| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | +| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name         = string
    subnetwork_self_link = string
    additional_networks = list(object({
      subnetwork = string
    }))
  })
| n/a | yes | +| [nodeset\_template](#input\_nodeset\_template) | Self link of the nodeset template | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf new file mode 100644 index 0000000000..bd8773cf84 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf @@ -0,0 +1,46 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + cleanup_dependencies_agg = flatten([ + var.nodeset.subnetwork_self_link, + var.nodeset.additional_networks[*].subnetwork, + var.nodeset_template]) +} + +# Can not use variadic list in `depends_on`, wrap it into a collection of `null_resource` +resource "null_resource" "dependencies" { + count = length(local.cleanup_dependencies_agg) +} + +resource "null_resource" "script" { + count = var.enable_cleanup_compute ? 1 : 0 + + triggers = { + project_id = var.project_id + cluster_name = var.slurm_cluster_name + nodeset_name = var.nodeset.nodeset_name + universe_domain = var.universe_domain + compute_endpoint_version = var.endpoint_versions.compute + gcloud_path_override = var.gcloud_path_override + } + + provisioner "local-exec" { + command = "/bin/bash ${path.module}/scripts/cleanup_compute.sh ${self.triggers.project_id} ${self.triggers.cluster_name} ${self.triggers.nodeset_name} ${self.triggers.universe_domain} ${self.triggers.compute_endpoint_version} ${self.triggers.gcloud_path_override}" + when = destroy + } + + # Ensure that clean up is done before attempt to delete the networks + depends_on = [null_resource.dependencies] +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh new file mode 100644 index 0000000000..20d611969a --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e -o pipefail + +project="$1" +cluster_name="$2" +nodeset_name="$3" +universe_domain="$4" +compute_endpoint_version="$5" +gcloud_dir="$6" + +if [[ $# -ne 5 ]] && [[ $# -ne 6 ]]; then + echo "Usage: $0 []" + exit 1 +fi + +if [[ -n "${gcloud_dir}" ]]; then + export PATH="$gcloud_dir:$PATH" +fi + +export CLOUDSDK_API_ENDPOINT_OVERRIDES_COMPUTE="https://www.${universe_domain}/compute/${compute_endpoint_version}/" +export CLOUDSDK_CORE_PROJECT="${project}" + +if ! type -P gcloud 1>/dev/null; then + echo "gcloud is not available and your compute resources are not being cleaned up" + echo "https://console.cloud.google.com/compute/instances?project=${project}" + exit 1 +fi + +tmpfile=$(mktemp) # have to use a temp file, since `< <(gcloud ...)` doesn't work nicely with `head` +trap 'rm -f "$tmpfile"' EXIT + +echo "Deleting managed instance groups" +mig_filter="name:${cluster_name}-${nodeset_name}-* AND status!=STOPPING" +gcloud compute instance-groups managed list --format="value(self_link)" --filter="${mig_filter}" >"$tmpfile" +while batch="$(head -n 2)" && [[ ${#batch} -gt 0 ]]; do + groups=$(echo "$batch" | paste -sd " " -) # concat into a single space-separated line + # The lack of quotes around ${groups} is intentional and causes each new space-separated "word" to + # be treated as independent arguments. See PR#2523 + # shellcheck disable=SC2086 + gcloud compute instance-groups managed delete --quiet ${groups} || echo "Failed to delete some instance groups" +done <"$tmpfile" +true >"$tmpfile" # Wipe contents of tmp file + +echo "Deleting compute nodes" +node_filter="name:${cluster_name}-${nodeset_name}-* labels.slurm_cluster_name=${cluster_name} AND labels.slurm_instance_role=compute" + +running_nodes_filter="${node_filter} AND status!=STOPPING" +# List all currently running instances and attempt to delete them +gcloud compute instances list --format="value(selfLink)" --filter="${running_nodes_filter}" >"$tmpfile" +# Do 500 instances at a time +while batch="$(head -n 500)" && [[ ${#batch} -gt 0 ]]; do + nodes=$(echo "$batch" | paste -sd " " -) # concat into a single space-separated line + # The lack of quotes around ${nodes} is intentional and causes each new space-separated "word" to + # be treated as independent arguments. See PR#2523 + # shellcheck disable=SC2086 + gcloud compute instances delete --quiet ${nodes} || echo "Failed to delete some instances" +done <"$tmpfile" + +# In case if controller tries to delete the nodes as well, +# wait until nodes in STOPPING state are deleted, before deleting the resource policies +stopping_nodes_filter="${node_filter} AND status=STOPPING" +while true; do + node=$(gcloud compute instances list --format="value(name)" --filter="${stopping_nodes_filter}" --limit=1) + if [[ -z "${node}" ]]; then + break + fi + echo "Waiting for instances to be deleted: ${node}" + sleep 5 +done + +echo "Deleting resource policies" +policies_filter="name:${cluster_name}-slurmgcp-managed-${nodeset_name}-*" +gcloud compute resource-policies list --format="value(selfLink)" --filter="${policies_filter}" | while read -r line; do + echo "Deleting resource policy: $line" + gcloud compute resource-policies delete --quiet "${line}" || { + echo "Failed to delete resource policy: $line" + } +done diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf new file mode 100644 index 0000000000..b6da69931c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf @@ -0,0 +1,71 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "Project ID" +} + + +variable "slurm_cluster_name" { + type = string + description = "Name of the Slurm cluster" +} + +variable "enable_cleanup_compute" { + description = < [terraform](#requirement\_terraform) | >= 1.3 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [null](#provider\_null) | 3.2.3 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of TPU nodes managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed TPU nodes will be destroyed. | `bool` | n/a | yes | +| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | +| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | +| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name = string
    zone         = string
  })
| n/a | yes | +| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | + +## Outputs + +No outputs. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [null](#provider\_null) | >= 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of TPU nodes managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed TPU nodes will be destroyed. | `bool` | n/a | yes | +| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | +| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | +| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name = string
    zone         = string
  })
| n/a | yes | +| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | +| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf new file mode 100644 index 0000000000..ec86a03a24 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf @@ -0,0 +1,32 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "null_resource" "script" { + count = var.enable_cleanup_compute ? 1 : 0 + + triggers = { + project_id = var.project_id + cluster_name = var.slurm_cluster_name + nodeset_name = var.nodeset.nodeset_name + zone = var.nodeset.zone + universe_domain = var.universe_domain + compute_endpoint_version = var.endpoint_versions.compute + gcloud_path_override = var.gcloud_path_override + } + + provisioner "local-exec" { + command = "/bin/bash ${path.module}/scripts/cleanup_tpu.sh ${self.triggers.project_id} ${self.triggers.cluster_name} ${self.triggers.nodeset_name} ${self.triggers.zone} ${self.triggers.universe_domain} ${self.triggers.compute_endpoint_version} ${self.triggers.gcloud_path_override}" + when = destroy + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh new file mode 100644 index 0000000000..c724e342c3 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e -o pipefail + +project="$1" +cluster_name="$2" +nodeset_name="$3" +zone="$4" +universe_domain="$5" +compute_endpoint_version="$6" +gcloud_dir="$7" + +if [[ $# -ne 6 ]] && [[ $# -ne 7 ]]; then + echo "Usage: $0 []" + exit 1 +fi + +if [[ -n "${gcloud_dir}" ]]; then + export PATH="$gcloud_dir:$PATH" +fi + +export CLOUDSDK_API_ENDPOINT_OVERRIDES_COMPUTE="https://www.${universe_domain}/compute/${compute_endpoint_version}/" +export CLOUDSDK_CORE_PROJECT="${project}" + +if ! type -P gcloud 1>/dev/null; then + echo "gcloud is not available and your compute resources are not being cleaned up" + echo "https://console.cloud.google.com/compute/instances?project=${project}" + exit 1 +fi + +echo "Deleting TPU nodes" +node_filter="name~${cluster_name}-${nodeset_name}" +running_nodes_filter="${node_filter} AND state!=DELETING" + +# List all currently running nodes and attempt to delete them +gcloud compute tpus tpu-vm list --zone="${zone}" --format="value(name)" --filter="${running_nodes_filter}" | while read -r name; do + echo "Deleting TPU node: $name" + gcloud compute tpus tpu-vm delete --async --zone="${zone}" --quiet "${name}" || echo "Failed to delete $name" +done + +# Wait until nodes in DELETING state are deleted, before deleting the resource policies +deleting_nodes_filter="${node_filter} AND state=DELETING" +while true; do + node=$(gcloud compute tpus tpu-vm list --zone="${zone}" --format="value(name)" --filter="${deleting_nodes_filter}" --limit=1) + if [[ -z "${node}" ]]; then + break + fi + echo "Waiting for nodes to be deleted: ${node}" + sleep 5 +done diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf new file mode 100644 index 0000000000..1ac6f64b75 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf @@ -0,0 +1,60 @@ +/** + * Copyright (C) Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "Project ID" +} + +variable "slurm_cluster_name" { + type = string + description = "Name of the Slurm cluster" +} + +variable "enable_cleanup_compute" { + description = < +Copyright (C) SchedMD LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~> 1.3 | +| [archive](#requirement\_archive) | ~> 2.0 | +| [google](#requirement\_google) | >= 3.53 | +| [local](#requirement\_local) | ~> 2.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [archive](#provider\_archive) | ~> 2.0 | +| [google](#provider\_google) | >= 3.53 | +| [local](#provider\_local) | ~> 2.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket_object.config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.controller_startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.devel](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.epilog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.nodeset_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.nodeset_dyn_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.nodeset_startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.nodeset_tpu_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.prolog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.task_epilog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_storage_bucket_object.task_prolog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [random_uuid.cluster_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/uuid) | resource | +| [archive_file.slurm_gcp_devel_zip](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | +| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | +| [local_file.chs_gpu_health_check](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | +| [local_file.external_epilog](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | +| [local_file.external_prolog](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | +| [local_file.setup_external](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [bucket\_dir](#input\_bucket\_dir) | Bucket directory for cluster files to be put into. | `string` | `null` | no | +| [bucket\_name](#input\_bucket\_name) | Name of GCS bucket to use. | `string` | n/a | yes | +| [cgroup\_conf\_tpl](#input\_cgroup\_conf\_tpl) | Slurm cgroup.conf template file path. | `string` | `null` | no | +| [cloud\_parameters](#input\_cloud\_parameters) | cloud.conf options. Default behavior defined in scripts/conf.py | 
object({
    no_comma_params      = optional(bool, false)
    private_data         = optional(list(string))
    scheduler_parameters = optional(list(string))
    resume_rate          = optional(number)
    resume_timeout       = optional(number)
    suspend_rate         = optional(number)
    suspend_timeout      = optional(number)
    topology_plugin      = optional(string)
    topology_param       = optional(string)
    tree_width           = optional(number)
  })
| `{}` | no | +| [cloudsql\_secret](#input\_cloudsql\_secret) | Secret URI to cloudsql secret. | `string` | `null` | no | +| [compute\_startup\_scripts\_timeout](#input\_compute\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in compute\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [controller\_network\_attachment](#input\_controller\_network\_attachment) | SelfLink for NetworkAttachment to be attached to the controller, if any. | `string` | `null` | no | +| [controller\_startup\_scripts](#input\_controller\_startup\_scripts) | List of scripts to be ran on controller VM startup. | 
list(object({
    filename = string
    content  = string
  }))
| `[]` | no | +| [controller\_startup\_scripts\_timeout](#input\_controller\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in controller\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | +| [controller\_state\_disk](#input\_controller\_state\_disk) | A disk that will be attached to the controller instance template to save state of slurm. The disk is created and used by default.
To disable this feature, set this variable to null.

NOTE: This will not save the contents at /opt/apps and /home. To preserve those, they must be saved externally. | 
object({
    device_name = string
  })
| 
{
  "device_name": null
}
| no | +| [disable\_default\_mounts](#input\_disable\_default\_mounts) | Disable default global network storage from the controller
- /home
- /apps | `bool` | `false` | no | +| [enable\_bigquery\_load](#input\_enable\_bigquery\_load) | Enables loading of cluster job usage into big query.

NOTE: Requires Google Bigquery API. | `bool` | `false` | no | +| [enable\_chs\_gpu\_health\_check\_epilog](#input\_enable\_chs\_gpu\_health\_check\_epilog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as an epilog script after completing a job step from a new job allocation.
Compute nodes that fail GPU health check during epilog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | +| [enable\_chs\_gpu\_health\_check\_prolog](#input\_enable\_chs\_gpu\_health\_check\_prolog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as a prolog script whenever it is asked to run a job step from a new job allocation. Compute nodes that fail GPU health check during prolog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | +| [enable\_debug\_logging](#input\_enable\_debug\_logging) | Enables debug logging mode. Not for production use. | `bool` | `false` | no | +| [enable\_external\_prolog\_epilog](#input\_enable\_external\_prolog\_epilog) | Automatically enable a script that will execute prolog and epilog scripts
shared by NFS from the controller to compute nodes. Find more details at:
https://github.com/GoogleCloudPlatform/slurm-gcp/blob/v5/tools/prologs-epilogs/README.md | `bool` | `false` | no | +| [enable\_hybrid](#input\_enable\_hybrid) | Enables use of hybrid controller mode. When true, controller\_hybrid\_config will
be used instead of controller\_instance\_config and will disable login instances. | `bool` | `false` | no | +| [enable\_slurm\_auth](#input\_enable\_slurm\_auth) | Enables slurm authentication instead of munge. | `bool` | `false` | no | +| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| 
{
  "compute": null
}
| no | +| [epilog\_scripts](#input\_epilog\_scripts) | List of scripts to be used for Epilog. Programs for the slurmd to execute
on every node when a user's job completes.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Epilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [extra\_logging\_flags](#input\_extra\_logging\_flags) | The only available flag is `trace_api` | `map(bool)` | `{}` | no | +| [google\_app\_cred\_path](#input\_google\_app\_cred\_path) | Path to Google Application Credentials. | `string` | `null` | no | +| [install\_dir](#input\_install\_dir) | Directory where the hybrid configuration directory will be installed on the
on-premise controller (e.g. /etc/slurm/hybrid). This updates the prefix path
for the resume and suspend scripts in the generated `cloud.conf` file.

This variable should be used when the TerraformHost and the SlurmctldHost
are different.

This will default to var.output\_dir if null. | `string` | `null` | no | +| [munge\_mount](#input\_munge\_mount) | Remote munge mount for compute and login nodes to acquire the munge.key.
By default, the munge mount server will be assumed to be the
`var.slurm_control_host` (or `var.slurm_control_addr` if non-null) when
`server_ip=null`. | 
object({
    server_ip     = string
    remote_mount  = string
    fs_type       = string
    mount_options = string
  })
| 
{
  "fs_type": "nfs",
  "mount_options": "",
  "remote_mount": "/etc/munge/",
  "server_ip": null
}
| no | +| [network\_storage](#input\_network\_storage) | Storage to mounted on all instances.
- server\_ip : Address of the storage server.
- remote\_mount : The location in the remote instance filesystem to mount from.
- local\_mount : The location on the instance filesystem to mount to.
- fs\_type : Filesystem type (e.g. "nfs").
- mount\_options : Options to mount with. | 
list(object({
    server_ip     = string
    remote_mount  = string
    local_mount   = string
    fs_type       = string
    mount_options = string
  }))
| `[]` | no | +| [nodeset](#input\_nodeset) | Cluster nodenets, as a list. | `list(any)` | `[]` | no | +| [nodeset\_dyn](#input\_nodeset\_dyn) | Cluster nodenets (dynamic), as a list. | `list(any)` | `[]` | no | +| [nodeset\_startup\_scripts](#input\_nodeset\_startup\_scripts) | List of scripts to be ran on compute VM startup in the specific nodeset. | 
map(list(object({
    filename = string
    content  = string
  })))
| `{}` | no | +| [nodeset\_tpu](#input\_nodeset\_tpu) | Cluster nodenets (TPU), as a list. | `list(any)` | `[]` | no | +| [output\_dir](#input\_output\_dir) | Directory where this module will write its files to. These files include:
cloud.conf; cloud\_gres.conf; config.yaml; resume.py; suspend.py; and util.py. | `string` | `null` | no | +| [project\_id](#input\_project\_id) | The GCP project ID. | `string` | n/a | yes | +| [prolog\_scripts](#input\_prolog\_scripts) | List of scripts to be used for Prolog. Programs for the slurmd to execute
whenever it is asked to run a job step from a new job allocation.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Prolog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [slurm\_bin\_dir](#input\_slurm\_bin\_dir) | Path to directory of Slurm binary commands (e.g. scontrol, sinfo). If 'null',
then it will be assumed that binaries are in $PATH. | `string` | `null` | no | +| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | The cluster name, used for resource naming and slurm accounting. | `string` | n/a | yes | +| [slurm\_conf\_template](#input\_slurm\_conf\_template) | Slurm slurm.conf template. Content of the file in 'slurm\_conf\_tpl' is used if this is not set. | `string` | `null` | no | +| [slurm\_conf\_tpl](#input\_slurm\_conf\_tpl) | Slurm slurm.conf template file path. This path is used only if raw content is not provided in 'slurm\_conf\_template'. | `string` | `null` | no | +| [slurm\_control\_addr](#input\_slurm\_control\_addr) | The IP address or a name by which the address can be identified.

This value is passed to slurm.conf such that:
SlurmctldHost={var.slurm\_control\_host}\({var.slurm\_control\_addr}\)

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldHost | `string` | `null` | no | +| [slurm\_control\_host](#input\_slurm\_control\_host) | The short, or long, hostname of the machine where Slurm control daemon is
executed (i.e. the name returned by the command "hostname -s").

This value is passed to slurm.conf such that:
SlurmctldHost={var.slurm\_control\_host}\({var.slurm\_control\_addr}\)

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldHost | `string` | `null` | no | +| [slurm\_control\_host\_port](#input\_slurm\_control\_host\_port) | The port number that the Slurm controller, slurmctld, listens to for work.

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldPort | `string` | `"6818"` | no | +| [slurm\_key\_mount](#input\_slurm\_key\_mount) | Remote mount for compute and login nodes to acquire the slurm.key. | 
object({
    server_ip     = string
    remote_mount  = string
    fs_type       = string
    mount_options = string
  })
| `null` | no | +| [slurm\_log\_dir](#input\_slurm\_log\_dir) | Directory where Slurm logs to. | `string` | `"/var/log/slurm"` | no | +| [slurmdbd\_conf\_tpl](#input\_slurmdbd\_conf\_tpl) | Slurm slurmdbd.conf template file path. | `string` | `null` | no | +| [task\_epilog\_scripts](#input\_task\_epilog\_scripts) | List of scripts to be used for TaskEpilog. Programs for the slurmd to execute
as the slurm job's owner after termination of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskEpilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | +| [task\_prolog\_scripts](#input\_task\_prolog\_scripts) | List of scripts to be used for TaskProlog. Programs for the slurmd to execute
as the slurm job's owner prior to initiation of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskProlog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [bucket\_dir](#output\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | +| [bucket\_name](#output\_bucket\_name) | GCS Bucket name of Slurm cluster file storage. | +| [config](#output\_config) | Cluster configuration. | +| [slurm\_bucket\_path](#output\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl new file mode 100644 index 0000000000..ffeb167cfc --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl @@ -0,0 +1,7 @@ +# cgroup.conf +# https://slurm.schedmd.com/cgroup.conf.html + +ConstrainCores=yes +ConstrainRamSpace=yes +ConstrainSwapSpace=no +ConstrainDevices=yes diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl new file mode 100644 index 0000000000..4951289842 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl @@ -0,0 +1,67 @@ +# slurm.conf +# https://slurm.schedmd.com/slurm.conf.html +# https://slurm.schedmd.com/configurator.html + +ProctrackType=proctrack/cgroup +SlurmctldPidFile=/var/run/slurm/slurmctld.pid +SlurmdPidFile=/var/run/slurm/slurmd.pid +TaskPlugin=task/affinity,task/cgroup +MaxNodeCount=64000 + +# +# +# SCHEDULING +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core_Memory + +# +# +# LOGGING AND ACCOUNTING +AccountingStoreFlags=job_comment +JobAcctGatherFrequency=30 +JobAcctGatherType=jobacct_gather/cgroup +SlurmctldDebug=info +SlurmdDebug=info +DebugFlags=Power + +# +# +# TIMERS +MessageTimeout=60 + +################################################################################ +# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # +################################################################################ + +SlurmctldHost={control_host}({control_addr}) + +AuthType=auth/{auth_key} +AuthInfo=cred_expire=120 +AuthAltTypes=auth/jwt +CredType=cred/{auth_key} +MpiDefault={mpi_default} +ReturnToService=2 +SlurmctldPort={control_host_port} +SlurmdPort=6818 +SlurmdSpoolDir=/var/spool/slurmd +SlurmUser=slurm +StateSaveLocation={state_save} + +# +# +# LOGGING AND ACCOUNTING +AccountingStorageType=accounting_storage/slurmdbd +AccountingStorageHost={accounting_storage_host} +ClusterName={name} +SlurmctldLogFile={slurmlog}/slurmctld.log +SlurmdLogFile={slurmlog}/slurmd-%n.log + +# +# +# GENERATED CLOUD CONFIGURATIONS +include cloud.conf + +################################################################################ +# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # +################################################################################ diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl new file mode 100644 index 0000000000..8c90a9dfbe --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl @@ -0,0 +1,31 @@ +# slurmdbd.conf +# https://slurm.schedmd.com/slurmdbd.conf.html + +DebugLevel=info +PidFile=/var/run/slurm/slurmdbd.pid + +################################################################################ +# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # +################################################################################ + +AuthType=auth/{auth_key} +AuthAltTypes=auth/jwt +AuthAltParameters=jwt_key={state_save}/jwt_hs256.key + +DbdHost={control_host} + +LogFile={slurmlog}/slurmdbd.log + +SlurmUser=slurm + +StorageLoc={db_name} + +StorageType=accounting_storage/mysql +StorageHost={db_host} +StoragePort={db_port} +StorageUser={db_user} +StoragePass={db_pass} + +################################################################################ +# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # +################################################################################ diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh new file mode 100644 index 0000000000..db514fc9e5 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -x /opt/apps/adm/slurm/slurm_epilog ]]; then + exec /opt/apps/adm/slurm/slurm_epilog +fi diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh new file mode 100644 index 0000000000..37a91bb1ea --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -x /opt/apps/adm/slurm/slurm_prolog ]]; then + exec /opt/apps/adm/slurm/slurm_prolog +fi diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh new file mode 100644 index 0000000000..21454bd52c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +SLURM_EXTERNAL_ROOT="/opt/apps/adm/slurm" +SLURM_MUX_FILE="slurm_mux" + +mkdir -p "${SLURM_EXTERNAL_ROOT}" +mkdir -p "${SLURM_EXTERNAL_ROOT}/logs" +mkdir -p "${SLURM_EXTERNAL_ROOT}/etc" + +# create common prolog / epilog "multiplex" script +if [ ! -f "${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" ]; then + # indentation matters in EOT below; do not blindly edit! + cat <<'EOT' >"${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CMD="${0##*/}" +# Locate script +BASE=$(readlink -f $0) +BASE=${BASE%/*} + +export CLUSTER_ADM_BASE=${BASE} + +# Source config file if it exists for extra DEBUG settings +# used below +SLURM_MUX_CONF=${CLUSTER_ADM_BASE}/etc/slurm_mux.conf +if [[ -r ${SLURM_MUX_CONF} ]]; then + source ${SLURM_MUX_CONF} +fi + +# Setup logging if configured and directory exists +LOGFILE="/dev/null" +if [[ -d ${DEBUG_SLURM_MUX_LOG_DIR} && ${DEBUG_SLURM_MUX_ENABLE_LOG} == "yes" ]]; then + LOGFILE="${DEBUG_SLURM_MUX_LOG_DIR}/${CMD}-${SLURM_SCRIPT_CONTEXT}-job-${SLURMD_NODENAME}.log" + exec >>${LOGFILE} 2>&1 +fi + +# Global scriptlets +for SCRIPTLET in ${BASE}/${SLURM_SCRIPT_CONTEXT}.d/*.${SLURM_SCRIPT_CONTEXT}; do + if [[ -x ${SCRIPTLET} ]]; then + echo "Running ${SCRIPTLET}" + ${SCRIPTLET} $@ >>${LOGFILE} 2>&1 + fi +done + +# Per partition scriptlets +for SCRIPTLET in ${BASE}/partition-${SLURM_JOB_PARTITION}-${SLURM_SCRIPT_CONTEXT}.d/*.${SLURM_SCRIPT_CONTEXT}; do + if [[ -x ${SCRIPTLET} ]]; then + echo "Running ${SCRIPTLET}" + ${SCRIPTLET} $@ >>${LOGFILE} 2>&1 + fi +done +EOT +fi + +# ensure proper permissions on slurm_mux script +chmod 0755 "${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" + +# create default slurm_mux configuration file +if [ ! -f "${SLURM_EXTERNAL_ROOT}/etc/slurm_mux.conf" ]; then + cat <<'EOT' >"${SLURM_EXTERNAL_ROOT}/etc/slurm_mux.conf" +# these settings are intended for temporary debugging purposes only; leaving +# them enabled will write files for each job to a shared NFS directory without +# any automated cleanup +DEBUG_SLURM_MUX_LOG_DIR=/opt/apps/adm/slurm/logs +DEBUG_SLURM_MUX_ENABLE_LOG=no +EOT +fi + +# create epilog symbolic link +if [ ! -L "${SLURM_EXTERNAL_ROOT}/slurm_epilog" ]; then + cd ${SLURM_EXTERNAL_ROOT} + # delete existing file if necessary + rm -f slurm_epilog + ln -s ${SLURM_MUX_FILE} slurm_epilog + cd - >/dev/null +fi + +# create prolog symbolic link +if [ ! -L "${SLURM_EXTERNAL_ROOT}/slurm_prolog" ]; then + cd ${SLURM_EXTERNAL_ROOT} + # delete existing file if necessary + rm -f slurm_prolog + ln -s ${SLURM_MUX_FILE} slurm_prolog + cd - >/dev/null +fi diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf new file mode 100644 index 0000000000..a4855cb67c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf @@ -0,0 +1,331 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + scripts_dir = abspath("${path.module}/scripts") + + bucket_dir = coalesce(var.bucket_dir, format("%s-files", var.slurm_cluster_name)) +} + +######## +# DATA # +######## + +data "google_storage_bucket" "this" { + name = var.bucket_name +} + +########## +# RANDOM # +########## + +resource "random_uuid" "cluster_id" { +} + +################## +# CLUSTER CONFIG # +################## + +locals { + config = { + enable_bigquery_load = var.enable_bigquery_load + cloudsql_secret = var.cloudsql_secret + cluster_id = random_uuid.cluster_id.result + project = var.project_id + slurm_cluster_name = var.slurm_cluster_name + enable_slurm_auth = var.enable_slurm_auth + bucket_path = local.bucket_path + enable_debug_logging = var.enable_debug_logging + extra_logging_flags = var.extra_logging_flags + controller_state_disk = var.controller_state_disk + + # storage + disable_default_mounts = var.disable_default_mounts + network_storage = var.network_storage + + # timeouts + controller_startup_scripts_timeout = var.controller_startup_scripts_timeout + compute_startup_scripts_timeout = var.compute_startup_scripts_timeout + + munge_mount = local.munge_mount + slurm_key_mount = var.slurm_key_mount + + # slurm conf + prolog_scripts = [for k, v in google_storage_bucket_object.prolog_scripts : k] + epilog_scripts = [for k, v in google_storage_bucket_object.epilog_scripts : k] + task_prolog_scripts = [for k, v in google_storage_bucket_object.task_prolog_scripts : k] + task_epilog_scripts = [for k, v in google_storage_bucket_object.task_epilog_scripts : k] + cloud_parameters = var.cloud_parameters + + # hybrid + hybrid = var.enable_hybrid + google_app_cred_path = var.enable_hybrid ? local.google_app_cred_path : null + output_dir = var.enable_hybrid ? local.output_dir : null + install_dir = var.enable_hybrid ? local.install_dir : null + slurm_control_host = var.enable_hybrid ? var.slurm_control_host : null + slurm_control_host_port = var.enable_hybrid ? local.slurm_control_host_port : null + slurm_control_addr = var.enable_hybrid ? var.slurm_control_addr : null + slurm_bin_dir = var.enable_hybrid ? local.slurm_bin_dir : null + slurm_log_dir = var.enable_hybrid ? local.slurm_log_dir : null + controller_network_attachment = var.controller_network_attachment + + + # config files templates + slurmdbd_conf_tpl = file(coalesce(var.slurmdbd_conf_tpl, "${local.etc_dir}/slurmdbd.conf.tpl")) + slurm_conf_tpl = var.slurm_conf_template != null ? var.slurm_conf_template : file(coalesce(var.slurm_conf_tpl, "${local.etc_dir}/slurm.conf.tpl")) + cgroup_conf_tpl = file(coalesce(var.cgroup_conf_tpl, "${local.etc_dir}/cgroup.conf.tpl")) + + # Providers + endpoint_versions = var.endpoint_versions + } + + x_nodeset = toset(var.nodeset[*].nodeset_name) + x_nodeset_dyn = toset(var.nodeset_dyn[*].nodeset_name) + x_nodeset_tpu = toset(var.nodeset_tpu[*].nodeset.nodeset_name) + x_nodeset_overlap = setintersection([], local.x_nodeset, local.x_nodeset_dyn, local.x_nodeset_tpu) + + etc_dir = abspath("${path.module}/etc") + + bucket_path = format("%s/%s", data.google_storage_bucket.this.url, local.bucket_dir) + + slurm_control_host_port = coalesce(var.slurm_control_host_port, "6818") + + google_app_cred_path = var.google_app_cred_path != null ? abspath(var.google_app_cred_path) : null + slurm_bin_dir = var.slurm_bin_dir != null ? abspath(var.slurm_bin_dir) : null + slurm_log_dir = var.slurm_log_dir != null ? abspath(var.slurm_log_dir) : null + + munge_mount = var.enable_hybrid ? { + server_ip = lookup(var.munge_mount, "server_ip", coalesce(var.slurm_control_addr, var.slurm_control_host)) + remote_mount = lookup(var.munge_mount, "remote_mount", "/etc/munge/") + fs_type = lookup(var.munge_mount, "fs_type", "nfs") + mount_options = lookup(var.munge_mount, "mount_options", "") + } : null + + output_dir = can(coalesce(var.output_dir)) ? abspath(var.output_dir) : abspath(".") + install_dir = can(coalesce(var.install_dir)) ? abspath(var.install_dir) : local.output_dir +} + +resource "google_storage_bucket_object" "config" { + bucket = data.google_storage_bucket.this.name + name = "${local.bucket_dir}/config.yaml" + content = yamlencode(local.config) + + # Take dependency on all other "config artifacts" so creation of `config.yaml` + # can be used as a signal for setup.py that "everything is ready". + # Some of following files, particularly mount scripts for new NFSes, can take a while to be created. + depends_on = [ + google_storage_bucket_object.controller_startup_scripts, + google_storage_bucket_object.nodeset_startup_scripts, + google_storage_bucket_object.prolog_scripts, + google_storage_bucket_object.epilog_scripts, + google_storage_bucket_object.task_prolog_scripts, + google_storage_bucket_object.task_epilog_scripts + ] +} + +resource "google_storage_bucket_object" "nodeset_config" { + for_each = { for ns in var.nodeset : ns.nodeset_name => merge(ns, { + instance_properties = jsondecode(ns.instance_properties_json) + }) } + + bucket = data.google_storage_bucket.this.name + name = "${local.bucket_dir}/nodeset_configs/${each.key}.yaml" + content = yamlencode(each.value) +} + +resource "google_storage_bucket_object" "nodeset_dyn_config" { + for_each = { for ns in var.nodeset_dyn : ns.nodeset_name => ns } + + bucket = data.google_storage_bucket.this.name + name = "${local.bucket_dir}/nodeset_dyn_configs/${each.key}.yaml" + content = yamlencode(each.value) +} + +resource "google_storage_bucket_object" "nodeset_tpu_config" { + for_each = { for n in var.nodeset_tpu[*].nodeset : n.nodeset_name => n } + + bucket = data.google_storage_bucket.this.name + name = "${local.bucket_dir}/nodeset_tpu_configs/${each.key}.yaml" + content = yamlencode(each.value) +} + +######### +# DEVEL # +######### + +locals { + build_dir = abspath("${path.module}/build") + + slurm_gcp_devel_zip = "slurm-gcp-devel.zip" + slurm_gcp_devel_zip_bucket = format("%s/%s", local.bucket_dir, local.slurm_gcp_devel_zip) +} + +data "archive_file" "slurm_gcp_devel_zip" { + output_path = "${local.build_dir}/${local.slurm_gcp_devel_zip}" + type = "zip" + source_dir = local.scripts_dir + + excludes = flatten([ + fileset(local.scripts_dir, "tests/**"), + # TODO: consider removing (including nested) __pycache__ and all .* files + # Though it only affects developers + ]) + +} + +resource "google_storage_bucket_object" "devel" { + bucket = var.bucket_name + name = local.slurm_gcp_devel_zip_bucket + source = data.archive_file.slurm_gcp_devel_zip.output_path +} + + +########### +# SCRIPTS # +########### + +resource "google_storage_bucket_object" "controller_startup_scripts" { + for_each = { + for x in local.controller_startup_scripts + : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x + } + + bucket = var.bucket_name + name = format("%s/slurm-controller-script-%s", local.bucket_dir, each.key) + content = each.value.content +} + +resource "google_storage_bucket_object" "nodeset_startup_scripts" { + for_each = { for x in flatten([ + for nodeset, scripts in var.nodeset_startup_scripts + : [for s in scripts + : { + content = s.content, + name = format("slurm-nodeset-%s-script-%s", nodeset, replace(basename(s.filename), "/[^a-zA-Z0-9-_]/", "_")) } + ]]) : x.name => x.content } + + bucket = var.bucket_name + name = format("%s/%s", local.bucket_dir, each.key) + content = each.value +} + +resource "google_storage_bucket_object" "prolog_scripts" { + for_each = { + for x in local.prolog_scripts + : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x + } + + bucket = var.bucket_name + name = format("%s/slurm-prolog-script-%s", local.bucket_dir, each.key) + content = each.value.content + source = each.value.source +} + +resource "google_storage_bucket_object" "epilog_scripts" { + for_each = { + for x in local.epilog_scripts + : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x + } + + bucket = var.bucket_name + name = format("%s/slurm-epilog-script-%s", local.bucket_dir, each.key) + content = each.value.content + source = each.value.source +} + +resource "google_storage_bucket_object" "task_prolog_scripts" { + for_each = { + for x in local.task_prolog_scripts + : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x + } + + bucket = var.bucket_name + name = format("%s/slurm-task_prolog-script-%s", local.bucket_dir, each.key) + content = each.value.content + source = each.value.source +} + +resource "google_storage_bucket_object" "task_epilog_scripts" { + for_each = { + for x in local.task_epilog_scripts + : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x + } + + bucket = var.bucket_name + name = format("%s/slurm-task_epilog-script-%s", local.bucket_dir, each.key) + content = each.value.content + source = each.value.source +} + +############################ +# DATA: CHS GPU HEALTH CHECK +############################ + +data "local_file" "chs_gpu_health_check" { + filename = "${path.module}/scripts/tools/gpu-test" +} + +################################ +# DATA: EXTERNAL PROLOG/EPILOG # +################################ + +data "local_file" "external_epilog" { + filename = "${path.module}/files/external_epilog.sh" +} + +data "local_file" "external_prolog" { + filename = "${path.module}/files/external_prolog.sh" +} + +data "local_file" "setup_external" { + filename = "${path.module}/files/setup_external.sh" +} + +locals { + external_epilog = [{ + filename = "z_external_epilog.sh" + content = data.local_file.external_epilog.content + source = null + }] + external_prolog = [{ + filename = "z_external_prolog.sh" + content = data.local_file.external_prolog.content + source = null + }] + setup_external = [{ + filename = "z_setup_external.sh" + content = data.local_file.setup_external.content + }] + chs_gpu_health_check = [{ + filename = "a_chs_gpu_health_check.sh" + content = data.local_file.chs_gpu_health_check.content + source = null + }] + + chs_prolog = var.enable_chs_gpu_health_check_prolog ? local.chs_gpu_health_check : [] + ext_prolog = var.enable_external_prolog_epilog ? local.external_prolog : [] + prolog_scripts = concat(local.chs_prolog, local.ext_prolog, var.prolog_scripts) + task_prolog_scripts = var.task_prolog_scripts + + chs_epilog = var.enable_chs_gpu_health_check_epilog ? local.chs_gpu_health_check : [] + ext_epilog = var.enable_external_prolog_epilog ? local.external_epilog : [] + epilog_scripts = concat(local.chs_epilog, local.ext_epilog, var.epilog_scripts) + task_epilog_scripts = var.task_epilog_scripts + + controller_startup_scripts = var.enable_external_prolog_epilog ? concat(local.setup_external, var.controller_startup_scripts) : var.controller_startup_scripts + + +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf new file mode 100644 index 0000000000..111c997d62 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf @@ -0,0 +1,45 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "slurm_bucket_path" { + description = "GCS Bucket URI of Slurm cluster file storage." + value = local.bucket_path +} + +output "bucket_name" { + description = "GCS Bucket name of Slurm cluster file storage." + value = data.google_storage_bucket.this.name +} + +output "bucket_dir" { + description = "Path directory within `bucket_name` for Slurm cluster file storage." + value = local.bucket_dir +} + +output "config" { + description = "Cluster configuration." + value = local.config + + precondition { + condition = var.enable_hybrid ? can(coalesce(var.slurm_control_host)) : true + error_message = "Input slurm_control_host is required." + } + + precondition { + condition = length(local.x_nodeset_overlap) == 0 + error_message = "All nodeset names must be unique among all nodeset types." + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py new file mode 100644 index 0000000000..e063093800 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py @@ -0,0 +1,641 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Optional, Iterable, Dict, Set, Tuple +from itertools import chain +from collections import defaultdict +import json +from pathlib import Path +import util +from util import dirs, slurmdirs +import tpu +from addict import Dict as NSDict # type: ignore + +FILE_PREAMBLE = """ +# Warning: +# This file is managed by a script. Manual modifications will be overwritten. +""" + + + +def dict_to_conf(conf, delim=" ") -> str: + """convert dict to delimited slurm-style key-value pairs""" + + def filter_conf(pair): + k, v = pair + if isinstance(v, list): + v = ",".join(str(el) for el in v if el is not None) + return k, (v if bool(v) or v == 0 else None) + + return delim.join( + f"{k}={v}" for k, v in map(filter_conf, conf.items()) if v is not None + ) + + +TOPOLOGY_PLUGIN_TREE = "topology/tree" + +def topology_plugin(lkp: util.Lookup) -> str: + """ + Returns configured topology plugin, defaults to `topology/tree`. + """ + cp, key = lkp.cfg.cloud_parameters, "topology_plugin" + if key not in cp or cp[key] is None: + return TOPOLOGY_PLUGIN_TREE + return cp[key] + +def conflines(lkp: util.Lookup) -> str: + params = lkp.cfg.cloud_parameters + def get(key, default): + """ + Returns the value of the key in params if it exists and is not None, + otherwise returns supplied default. + We can't rely on the `dict.get` method because the value could be `None` as + well as empty NSDict, depending on type of the `cfg.cloud_parameters`. + TODO: Simplify once NSDict is removed from the codebase. + """ + if key not in params or params[key] is None: + return default + return params[key] + + no_comma_params = get("no_comma_params", False) + + any_gpus = any( + lkp.template_info(nodeset.instance_template).gpu + for nodeset in lkp.cfg.nodeset.values() + ) + + any_tpu = any( + tpu_nodeset is not None + for part in lkp.cfg.partitions.values() + for tpu_nodeset in part.partition_nodeset_tpu + ) + + any_dynamic = any(bool(p.partition_feature) for p in lkp.cfg.partitions.values()) + comma_params = { + "LaunchParameters": [ + "enable_nss_slurm", + "use_interactive_step", + ], + "SlurmctldParameters": [ + "cloud_reg_addrs" if any_dynamic or any_tpu else "cloud_dns", + "enable_configless", + "idle_on_node_suspend", + ], + "GresTypes": [ + "gpu" if any_gpus else None, + ], + } + + scripts_dir = lkp.cfg.install_dir or dirs.scripts + prolog_path = Path(dirs.custom_scripts / "prolog.d") + epilog_path = Path(dirs.custom_scripts / "epilog.d") + task_prolog_path = Path(dirs.custom_scripts / "task_prolog.d") + task_epilog_path = Path(dirs.custom_scripts / "task_epilog.d") + default_tree_width = 65533 if any_dynamic else 128 + + conf_options = { + **(comma_params if not no_comma_params else {}), + "Prolog": f"{prolog_path}/*" if lkp.cfg.prolog_scripts else None, + "Epilog": f"{epilog_path}/*" if lkp.cfg.epilog_scripts else None, + "TaskProlog": f"{task_prolog_path}/task-prolog" if lkp.cfg.task_prolog_scripts else None, + "TaskEpilog": f"{task_epilog_path}/task-epilog" if lkp.cfg.task_epilog_scripts else None, + "PrivateData": get("private_data", []), + "SchedulerParameters": get("scheduler_parameters", [ + "bf_continue", + "salloc_wait_nodes", + "ignore_prefer_validation", + ]), + "ResumeProgram": f"{scripts_dir}/resume_wrapper.sh", + "ResumeFailProgram": f"{scripts_dir}/suspend_wrapper.sh", + "ResumeRate": get("resume_rate", 0), + "ResumeTimeout": get("resume_timeout", 300), + "SuspendProgram": f"{scripts_dir}/suspend_wrapper.sh", + "SuspendRate": get("suspend_rate", 0), + "SuspendTimeout": get("suspend_timeout", 300), + "TreeWidth": get("tree_width", default_tree_width), + "JobSubmitPlugins": "lua" if any_tpu else None, + "TopologyPlugin": topology_plugin(lkp), + "TopologyParam": get("topology_param", "SwitchAsNodeRank"), + } + return dict_to_conf(conf_options, delim="\n") + + + + +def nodeset_lines(nodeset, lkp: util.Lookup) -> str: + template_info = lkp.template_info(nodeset.instance_template) + machine_conf = lkp.template_machine_conf(nodeset.instance_template) + + # follow https://slurm.schedmd.com/slurm.conf.html#OPT_Boards + # by setting Boards, SocketsPerBoard, CoresPerSocket, and ThreadsPerCore + gres = f"gpu:{template_info.gpu.count}" if template_info.gpu else None + node_conf = { + "RealMemory": machine_conf.memory, + "Boards": machine_conf.boards, + "SocketsPerBoard": machine_conf.sockets_per_board, + "CoresPerSocket": machine_conf.cores_per_socket, + "ThreadsPerCore": machine_conf.threads_per_core, + "CPUs": machine_conf.cpus, + "Gres": gres, + **nodeset.node_conf, + } + nodelist = lkp.nodelist(nodeset) + + return "\n".join( + map( + dict_to_conf, + [ + {"NodeName": nodelist, "State": "CLOUD", **node_conf}, + {"NodeSet": nodeset.nodeset_name, "Nodes": nodelist}, + ], + ) + ) + + +def nodeset_tpu_lines(nodeset, lkp: util.Lookup) -> str: + nodelist = lkp.nodelist(nodeset) + return "\n".join( + map( + dict_to_conf, + [ + {"NodeName": nodelist, "State": "CLOUD", **nodeset.node_conf}, + {"NodeSet": nodeset.nodeset_name, "Nodes": nodelist}, + ], + ) + ) + + +def nodeset_dyn_lines(nodeset): + """generate slurm NodeSet definition for dynamic nodeset""" + return dict_to_conf( + {"NodeSet": nodeset.nodeset_name, "Feature": nodeset.nodeset_feature} + ) + + +def partitionlines(partition, lkp: util.Lookup) -> str: + """Make a partition line for the slurm.conf""" + MIN_MEM_PER_CPU = 100 + + def defmempercpu(nodeset_name: str) -> int: + nodeset = lkp.cfg.nodeset.get(nodeset_name) + template = nodeset.instance_template + machine = lkp.template_machine_conf(template) + mem_spec_limit = int(nodeset.node_conf.get("MemSpecLimit", 0)) + return max(MIN_MEM_PER_CPU, (machine.memory - mem_spec_limit) // machine.cpus) + + defmem = min( + map(defmempercpu, partition.partition_nodeset), default=MIN_MEM_PER_CPU + ) + + nodesets = list( + chain( + partition.partition_nodeset, + partition.partition_nodeset_dyn, + partition.partition_nodeset_tpu, + ) + ) + + is_tpu = len(partition.partition_nodeset_tpu) > 0 + is_dyn = len(partition.partition_nodeset_dyn) > 0 + + oversub_exlusive = partition.enable_job_exclusive or is_tpu + power_down_on_idle = partition.enable_job_exclusive and not is_dyn + + line_elements = { + "PartitionName": partition.partition_name, + "Nodes": ",".join(nodesets), + "State": "UP", + "DefMemPerCPU": defmem, + "SuspendTime": 300, + "Oversubscribe": "Exclusive" if oversub_exlusive else None, + "PowerDownOnIdle": "YES" if power_down_on_idle else None, + **partition.partition_conf, + } + + return dict_to_conf(line_elements) + + +def suspend_exc_lines(lkp: util.Lookup) -> Iterable[str]: + static_nodelists = [] + for ns in lkp.power_managed_nodesets(): + if ns.node_count_static: + nodelist = lkp.nodelist_range(ns.nodeset_name, 0, ns.node_count_static) + static_nodelists.append(nodelist) + suspend_exc_nodes = {"SuspendExcNodes": static_nodelists} + + dyn_parts = [ + p.partition_name + for p in lkp.cfg.partitions.values() + if len(p.partition_nodeset_dyn) > 0 + ] + suspend_exc_parts = {"SuspendExcParts": [*dyn_parts]} + + return filter( + None, + [ + dict_to_conf(suspend_exc_nodes) if static_nodelists else None, + dict_to_conf(suspend_exc_parts), + ], + ) + + +def make_cloud_conf(lkp: util.Lookup) -> str: + """generate cloud.conf snippet""" + lines = [ + FILE_PREAMBLE, + conflines(lkp), + *(nodeset_lines(n, lkp) for n in lkp.cfg.nodeset.values()), + *(nodeset_dyn_lines(n) for n in lkp.cfg.nodeset_dyn.values()), + *(nodeset_tpu_lines(n, lkp) for n in lkp.cfg.nodeset_tpu.values()), + *(partitionlines(p, lkp) for p in lkp.cfg.partitions.values()), + *(suspend_exc_lines(lkp)), + ] + return "\n\n".join(filter(None, lines)) + + +def gen_cloud_conf(lkp: util.Lookup) -> None: + content = make_cloud_conf(lkp) + + conf_file = lkp.etc_dir / "cloud.conf" + conf_file.write_text(content) + util.chown_slurm(conf_file, mode=0o644) + + +def install_slurm_conf(lkp: util.Lookup) -> None: + """install slurm.conf""" + if lkp.cfg.ompi_version: + mpi_default = "pmi2" + else: + mpi_default = "none" + + conf_options = { + "name": lkp.cfg.slurm_cluster_name, + "control_addr": lkp.control_addr if lkp.control_addr else lkp.hostname_fqdn, + "control_host": lkp.control_host, + "accounting_storage_host": lkp.control_addr if lkp.cfg.controller_network_attachment else lkp.control_host, + "control_host_port": lkp.control_host_port, + "scripts": dirs.scripts, + "slurmlog": dirs.log, + "state_save": slurmdirs.state, + "mpi_default": mpi_default, + "auth_key": "slurm" if lkp.cfg.enable_slurm_auth else "munge", + } + + conf = lkp.cfg.slurm_conf_tpl.format(**conf_options) + + conf_file = lkp.etc_dir / "slurm.conf" + conf_file.write_text(conf) + util.chown_slurm(conf_file, mode=0o644) + + +def install_slurmdbd_conf(lkp: util.Lookup) -> None: + """install slurmdbd.conf""" + conf_options = { + "control_host": lkp.control_host, + "slurmlog": dirs.log, + "state_save": slurmdirs.state, + "db_name": "slurm_acct_db", + "db_user": "slurm", + "db_pass": '""', + "db_host": "localhost", + "db_port": "3306", + "auth_key": "slurm" if lkp.cfg.enable_slurm_auth else "munge", + } + + if lkp.cfg.cloudsql_secret: + secret_name = f"{lkp.cfg.slurm_cluster_name}-slurm-secret-cloudsql" + payload = json.loads(util.access_secret_version(lkp.project, secret_name)) + + if payload["db_name"] and payload["db_name"] != "": + conf_options["db_name"] = payload["db_name"] + if payload["user"] and payload["user"] != "": + conf_options["db_user"] = payload["user"] + if payload["password"] and payload["password"] != "": + conf_options["db_pass"] = payload["password"] + + db_host_str = payload["server_ip"].split(":") + if db_host_str[0]: + conf_options["db_host"] = db_host_str[0] + conf_options["db_port"] = ( + db_host_str[1] if len(db_host_str) >= 2 else "3306" + ) + + conf = lkp.cfg.slurmdbd_conf_tpl.format(**conf_options) + + conf_file = lkp.etc_dir / "slurmdbd.conf" + conf_file.write_text(conf) + util.chown_slurm(conf_file, 0o600) + + +def install_cgroup_conf(lkp: util.Lookup) -> None: + """install cgroup.conf""" + conf_file = lkp.etc_dir / "cgroup.conf" + conf_file.write_text(lkp.cfg.cgroup_conf_tpl) + util.chown_slurm(conf_file, mode=0o600) + + +def install_jobsubmit_lua(lkp: util.Lookup) -> None: + """install job_submit.lua if there are tpu nodes in the cluster""" + if not any( + tpu_nodeset is not None + for part in lkp.cfg.partitions.values() + for tpu_nodeset in part.partition_nodeset_tpu + ): + return # No TPU partitions, no need for job_submit.lua + + scripts_dir = lkp.cfg.slurm_scripts_dir or dirs.scripts + tpl = (scripts_dir / "job_submit.lua.tpl").read_text() + conf = tpl.format(scripts_dir=scripts_dir) + + conf_file = lkp.etc_dir / "job_submit.lua" + conf_file.write_text(conf) + util.chown_slurm(conf_file, 0o600) + + +def gen_cloud_gres_conf(lkp: util.Lookup) -> None: + """generate cloud_gres.conf""" + + gpu_nodes = defaultdict(list) + for nodeset in lkp.cfg.nodeset.values(): + ti = lkp.template_info(nodeset.instance_template) + gpu_count = ti.gpu.count if ti.gpu else 0 + if gpu_count: + gpu_nodes[gpu_count].append(lkp.nodelist(nodeset)) + + lines = [ + dict_to_conf( + { + "NodeName": names, + "Name": "gpu", + "File": "/dev/nvidia{}".format(f"[0-{i-1}]" if i > 1 else "0"), + } + ) + for i, names in gpu_nodes.items() + ] + lines.append("\n") + content = FILE_PREAMBLE + "\n".join(lines) + + conf_file = lkp.etc_dir / "cloud_gres.conf" + conf_file.write_text(content) + util.chown_slurm(conf_file, mode=0o600) + + +def install_gres_conf(lkp: util.Lookup) -> None: + conf_file = lkp.etc_dir / "cloud_gres.conf" + gres_conf = lkp.etc_dir / "gres.conf" + if not gres_conf.exists(): + gres_conf.symlink_to(conf_file) + util.chown_slurm(gres_conf, mode=0o600) + + +class Switch: + """ + Represents a switch in the topology.conf file. + NOTE: It's class user job to make sure that there is no leaf-less Switches in the tree + """ + + def __init__( + self, + name: str, + nodes: Optional[Iterable[str]] = None, + switches: Optional[Dict[str, "Switch"]] = None, + ): + self.name = name + self.nodes = nodes or [] + self.switches = switches or {} + + def conf_line(self) -> str: + d = {"SwitchName": self.name} + if self.nodes: + d["Nodes"] = util.to_hostlist(self.nodes) + if self.switches: + d["Switches"] = util.to_hostlist(self.switches.keys()) + return dict_to_conf(d) + + def render_conf_lines(self) -> Iterable[str]: + yield self.conf_line() + for s in sorted(self.switches.values(), key=lambda s: s.name): + yield from s.render_conf_lines() + +class TopologySummary: + """ + Represents a summary of the topology, to make judgements about changes. + To be stored in JSON file along side of topology.conf to simplify parsing. + """ + def __init__( + self, + physical_host: Optional[Dict[str, str]] = None, + down_nodes: Optional[Iterable[str]] = None, + tpu_nodes: Optional[Iterable[str]] = None, + ) -> None: + self.physical_host = physical_host or {} + self.down_nodes = set(down_nodes or []) + self.tpu_nodes = set(tpu_nodes or []) + + + @classmethod + def path(cls, lkp: util.Lookup) -> Path: + return lkp.etc_dir / "cloud_topology.summary.json" + + @classmethod + def loads(cls, s: str) -> "TopologySummary": + d = json.loads(s) + return cls( + physical_host=d.get("physical_host"), + down_nodes=d.get("down_nodes"), + tpu_nodes=d.get("tpu_nodes"), + ) + + @classmethod + def load(cls, lkp: util.Lookup) -> "TopologySummary": + p = cls.path(lkp) + if not p.exists(): + return cls() # Return empty instance + return cls.loads(p.read_text()) + + def dumps(self) -> str: + return json.dumps( + { + "physical_host": self.physical_host, + "down_nodes": list(self.down_nodes), + "tpu_nodes": list(self.tpu_nodes), + }, + indent=2) + + def dump(self, lkp: util.Lookup) -> None: + TopologySummary.path(lkp).write_text(self.dumps()) + + def _nodenames(self) -> Set[str]: + return set(self.physical_host) | self.down_nodes | self.tpu_nodes + + def requires_reconfigure(self, prev: "TopologySummary") -> bool: + """ + Reconfigure IFF one of the following occurs: + * A node is added + * A node get a non-empty physicalHost + """ + if len(self._nodenames() - prev._nodenames()) > 0: + return True + for n, ph in self.physical_host.items(): + if ph and ph != prev.physical_host.get(n): + return True + return False + +class TopologyBuilder: + def __init__(self) -> None: + self._r = Switch("") # fake root, not part of the tree + self.summary = TopologySummary() + + def add(self, path: List[str], nodes: Iterable[str]) -> None: + n = self._r + assert path + for p in path: + n = n.switches.setdefault(p, Switch(p)) + n.nodes = [*n.nodes, *nodes] + + def render_conf_lines(self) -> Iterable[str]: + if not self._r.switches: + return [] # type: ignore + for s in sorted(self._r.switches.values(), key=lambda s: s.name): + yield from s.render_conf_lines() + + def compress(self) -> "TopologyBuilder": + compressed = TopologyBuilder() + compressed.summary = self.summary + def _walk( + u: Switch, c: Switch + ): # u: uncompressed node, c: its counterpart in compressed tree + pref = f"{c.name}_" if c != compressed._r else "s" + for i, us in enumerate(sorted(u.switches.values(), key=lambda s: s.name)): + cs = Switch(f"{pref}{i}", nodes=us.nodes) + c.switches[cs.name] = cs + _walk(us, cs) + + _walk(self._r, compressed._r) + return compressed + + +def add_tpu_nodeset_topology(nodeset: NSDict, bldr: TopologyBuilder, lkp: util.Lookup): + tpuobj = tpu.TPU.make(nodeset.nodeset_name, lkp) + static, dynamic = lkp.nodenames(nodeset) + + pref = ["tpu-root", f"ns_{nodeset.nodeset_name}"] + if tpuobj.vmcount == 1: # Put all nodes in one switch + all_nodes = list(chain(static, dynamic)) + bldr.add(pref, all_nodes) + bldr.summary.tpu_nodes.update(all_nodes) + return + + # Chunk nodes into sub-switches of size `vmcount` + chunk_num = 0 + for nodenames in (static, dynamic): + for nodeschunk in util.chunked(nodenames, n=tpuobj.vmcount): + chunk_name = f"{nodeset.nodeset_name}-{chunk_num}" + chunk_num += 1 + bldr.add([*pref, chunk_name], nodeschunk) + bldr.summary.tpu_nodes.update(nodeschunk) + +_SLURM_TOPO_ROOT = "slurm-root" + +def _make_physical_path(physical_host: str) -> List[str]: + assert physical_host.startswith("/"), f"Unexpected physicalHost: {physical_host}" + parts = physical_host[1:].split("/") + # Due to issues with Slurm's topology plugin, we can not use all components of `physicalHost`, + # trim it down to `cluster/rack`. + short_path = parts[:2] + return [_SLURM_TOPO_ROOT, *short_path] + +def add_nodeset_topology( + nodeset: NSDict, bldr: TopologyBuilder, lkp: util.Lookup +) -> None: + up_nodes = set() + default_path = [_SLURM_TOPO_ROOT, f"ns_{nodeset.nodeset_name}"] + + for inst in lkp.instances().values(): + try: + if lkp.node_nodeset_name(inst.name) != nodeset.nodeset_name: + continue + except Exception: + continue + + phys_host = inst.resource_status.physical_host or "" + bldr.summary.physical_host[inst.name] = phys_host + up_nodes.add(inst.name) + + if phys_host: + bldr.add(_make_physical_path(phys_host), [inst.name]) + else: + bldr.add(default_path, [inst.name]) + + down_nodes = [] + for node in chain(*lkp.nodenames(nodeset)): + if node not in up_nodes: + down_nodes.append(node) + if down_nodes: + bldr.add(default_path, down_nodes) + bldr.summary.down_nodes.update(down_nodes) + +def gen_topology(lkp: util.Lookup) -> TopologyBuilder: + bldr = TopologyBuilder() + for ns in lkp.cfg.nodeset_tpu.values(): + add_tpu_nodeset_topology(ns, bldr, lkp) + for ns in lkp.cfg.nodeset.values(): + add_nodeset_topology(ns, bldr, lkp) + return bldr + +def gen_topology_conf(lkp: util.Lookup) -> Tuple[bool, TopologySummary]: + """ + Generates slurm topology.conf. + Returns whether the topology.conf got updated. + """ + topo = gen_topology(lkp).compress() + conf_file = lkp.etc_dir / "cloud_topology.conf" + + with open(conf_file, "w") as f: + f.writelines(FILE_PREAMBLE + "\n") + for line in topo.render_conf_lines(): + f.write(line) + f.write("\n") + f.write("\n") + + prev_summary = TopologySummary.load(lkp) + return topo.summary.requires_reconfigure(prev_summary), topo.summary + +def install_topology_conf(lkp: util.Lookup) -> None: + conf_file = lkp.etc_dir / "cloud_topology.conf" + summary_file = lkp.etc_dir / "cloud_topology.summary.json" + topo_conf = lkp.etc_dir / "topology.conf" + + if not topo_conf.exists(): + topo_conf.symlink_to(conf_file) + + util.chown_slurm(conf_file, mode=0o600) + util.chown_slurm(summary_file, mode=0o600) + + +def gen_controller_configs(lkp: util.Lookup) -> None: + install_slurm_conf(lkp) + install_slurmdbd_conf(lkp) + gen_cloud_conf(lkp) + gen_cloud_gres_conf(lkp) + install_gres_conf(lkp) + install_cgroup_conf(lkp) + install_jobsubmit_lua(lkp) + + if topology_plugin(lkp) == TOPOLOGY_PLUGIN_TREE: + _, summary = gen_topology_conf(lkp) + summary.dump(lkp) + install_topology_conf(lkp) diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py new file mode 100644 index 0000000000..cd2e41e5af --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py @@ -0,0 +1,80 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any +from pathlib import Path +import shutil +import pickle + +import logging +log = logging.getLogger() + +# Can't reuse tool from util.py to avoid circular dependencies +# TODO: break down util.py for better modularity. +def _chown_slurm(path: Path) -> None: + shutil.chown(path, user="slurm", group="slurm") + +class FileCache: + def __init__(self, path: Path): + self.path = path + + def get(self, key: str) -> Any | None: + p = self.path / key + if not p.exists(): + return None + + try: + with p.open("rb") as f: + return pickle.load(f) + + except Exception as e: + log.warning(f"Failed to read cached value at {p}: {e}") + return None + + def set(self, key: str, data: Any) -> None: + p = self.path / key + + try: + # Create & chown before writing to minimize chances + # of ending up with root-owned corrupted file that can't be cleaned up + # TODO: restrict usage of cache by root to avoid all this complexity + # or have a cache per user. + p.touch(exist_ok=True) + _chown_slurm(p) + with p.open("wb") as f: + pickle.dump(data, f) + + except Exception as e: + log.warning(f"Failed to write cached value at {p}: {e}") + + +class NoCache: + def get(self, key: str) -> Any: + log.warning("No cache used") + return None + + def set(self, key: str, data: Any) -> None: + log.warning("No cache used") + + +def cache(name: str) -> FileCache | NoCache: + try: + path = Path("/tmp/slurm_gcp_cache/") / name + if not path.exists(): + path.mkdir(exist_ok=True, parents=True) + _chown_slurm(path) + return FileCache(path) + except: + log.exception(f"Failed to create cache, fallback to NoCache") + return NoCache() diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py new file mode 100644 index 0000000000..df0fd8ebe0 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py @@ -0,0 +1,76 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright 2024 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import util +import tpu + + +def get_vmcount_of_tpu_part(part): + res = 0 + lkp = util.lookup() + for ns in lkp.cfg.partitions[part].partition_nodeset_tpu: + tpu_obj = tpu.TPU.make(ns, lkp) + if res == 0: + res = tpu_obj.vmcount + else: + if res != tpu_obj.vmcount: + # this should not happen, that in the same partition there are different vmcount nodesets + return -1 + return res + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--partitions", + "-p", + help="The partition(s) to retrieve the TPU vmcount value for.", + ) + args = parser.parse_args() + if not args.partitions: + exit(0) + + # useful exit code + # partition does not exists in config.yaml, thus do not exist in slurm + PART_INVALID = -1 + # in the same partition there are nodesets with different vmcounts + DIFF_VMCOUNTS_SAME_PART = -2 + # partition is a list of partitions in which at least two of them have different vmcount + DIFF_PART_DIFFERENT_VMCOUNTS = -3 + vmcounts = [] + # valid equals to 0 means that we are ok, otherwise it will be set to one of the previously defined exit codes + valid = 0 + for part in args.partitions.split(","): + if part not in util.lookup().cfg.partitions: + valid = PART_INVALID + break + else: + if util.lookup().partition_is_tpu(part): + vmcount = get_vmcount_of_tpu_part(part) + if vmcount == -1: + valid = DIFF_VMCOUNTS_SAME_PART + break + vmcounts.append(vmcount) + else: + vmcounts.append(0) + # this means that there are different vmcounts for these partitions + if valid == 0 and len(set(vmcounts)) != 1: + valid = DIFF_PART_DIFFERENT_VMCOUNTS + if valid != 0: + print(f"VMCOUNT:{valid}") + else: + print(f"VMCOUNT:{vmcounts[0]}") diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl new file mode 100644 index 0000000000..810a0742b0 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl @@ -0,0 +1,103 @@ +SCRIPTS_DIR = "{scripts_dir}" +NO_VAL = 4294967294 +-- get_tpu_vmcount.py exit code +PART_INVALID = -1 -- partition does not exists in config.yaml, thus do not exist in slurm +DIFF_VMCOUNTS_SAME_PART = -2 -- in the same partition there are nodesets with different vmcounts +DIFF_PART_DIFFERENT_VMCOUNTS = -3 -- partition is a list of partitions in which at least two of them have different vmcount +UNKWOWN_ERROR = -4 -- get_tpu_vmcount.py did not return a valid response + +function get_part(job_desc, part_list) + if job_desc.partition then + return job_desc.partition + end + for name, val in pairs(part_list) do + if val.flag_default == 1 then + return name + end + end + return nil +end + +function os.capture(cmd, raw) + local handle = assert(io.popen(cmd, 'r')) + local output = assert(handle:read('*a')) + handle:close() + return output +end + +function get_vmcount(part) + local cmd = SCRIPTS_DIR .. "/get_tpu_vmcount.py -p " .. part + local out = os.capture(cmd, true) + for line in out:gmatch("(.-)\r?\n") do + local tag, val = line:match("([^:]+):([^:]+)") + if tag == "VMCOUNT" then + return tonumber(val) + end + end + return UNKWOWN_ERROR +end + +function slurm_job_submit(job_desc, part_list, submit_uid) + local part = get_part(job_desc, part_list) + local vmcount = get_vmcount(part) + -- Only do something if the job is in a TPU partition, if vmcount is 0, it implies that the partition(s) specified are not TPU ones + if vmcount == 0 then + return slurm.SUCCESS + end + -- This is a TPU job, but as the vmcount is 1 it can he handled the same way + if vmcount == 1 then + return slurm.SUCCESS + end + -- Check for errors + if vmcount == PART_INVALID then + slurm.log_user("Invalid partition specified " .. part) + return slurm.FAILURE + end + if vmcount == DIFF_VMCOUNTS_SAME_PART then + slurm.log_user("In partition(s) " .. part .. + " there are more than one tpu nodeset vmcount, this should not happen.") + return slurm.ERROR + end + if vmcount == DIFF_PART_DIFFERENT_VMCOUNTS then + slurm.log_user("In partition list " .. part .. + " there are more than one TPU types, cannot determine which is the correct vmcount to use, please retry with only one partition.") + return slurm.FAILURE + end + if vmcount == UNKWOWN_ERROR then + slurm.log_user("Something went wrong while executing get_tpu_vmcount.py.") + return slurm.ERROR + end + -- This is surely a TPU node + if vmcount > 1 then + local min_nodes = job_desc.min_nodes + local max_nodes = job_desc.max_nodes + -- if not specified assume it is one, this should be improved taking into account the cpus, mem, and other factors + if min_nodes == NO_VAL then + min_nodes = 1 + max_nodes = 1 + end + -- as max_nodes can be higher than the nodes in the partition, we are not able to calculate with certainty the nodes that this job will have if this value is set to something + -- different than min_nodes + if min_nodes ~= max_nodes then + slurm.log_user("Max nodes cannot be set different than min nodes for the TPU partitions.") + return slurm.ERROR + end + -- Set the number of switches to the number of nodes originally requested by the job, as the job requests "TPU groups" + job_desc.req_switch = min_nodes + + -- Apply the node increase into the job description. + job_desc.min_nodes = min_nodes * vmcount + job_desc.max_nodes = max_nodes * vmcount + -- if job_desc.features then + -- slurm.log_user("Features: %s",job_desc.features) + -- end + end + + return slurm.SUCCESS +end + +function slurm_job_modify(job_desc, job_rec, part_list, modify_uid) + return slurm.SUCCESS +end + +return slurm.SUCCESS diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py new file mode 100644 index 0000000000..cabd6e3e9f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py @@ -0,0 +1,352 @@ +#!/slurm/python/venv/bin/python3.13 +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Callable, Any +import argparse +import os +import shelve +import uuid +from collections import namedtuple +from datetime import datetime, timedelta, timezone +from pathlib import Path +from pprint import pprint + +import util +from google.api_core import exceptions, retry +from google.cloud import bigquery as bq +from google.cloud.bigquery import SchemaField # type: ignore +from util import lookup, run + +SACCT = "sacct" +script = Path(__file__).resolve() + +DEFAULT_TIMESTAMP_FILE = script.parent / "bq_timestamp" +timestamp_file = Path(os.environ.get("TIMESTAMP_FILE", DEFAULT_TIMESTAMP_FILE)) +# The maximum request to insert_rows is 10MB, each sacct row is about 1200 bytes or ~ 8000 rows. +# Set to 5000 for a little wiggle room. +BQ_ROW_BATCH_SIZE = 5000 + +# cluster_id_file = script.parent / 'cluster_uuid' +# try: +# cluster_id = cluster_id_file.read_text().rstrip() +# except FileNotFoundError: +# cluster_id = uuid.uuid4().hex +# cluster_id_file.write_text(cluster_id) + +job_idx_cache_path = script.parent / "bq_job_idx_cache" + +SLURM_TIME_FORMAT = r"%Y-%m-%dT%H:%M:%S" + + +def make_datetime(time_string): + if time_string == "None": + return None + return datetime.strptime(time_string, SLURM_TIME_FORMAT).replace( + tzinfo=timezone.utc + ) + + +def make_time_interval(seconds): + sign = 1 + if seconds < 0: + sign = -1 + seconds = abs(seconds) + d, r = divmod(seconds, 60 * 60 * 24) + h, r = divmod(r, 60 * 60) + m, s = divmod(r, 60) + d *= sign + h *= sign + return f"{d}D {h:02}:{m:02}:{s}" + + +converters: Dict[str, Callable[[Any], Any]] = { + "DATETIME": make_datetime, + "INTERVAL": make_time_interval, + "STRING": str, + "INT64": lambda n: int(n or 0), +} + + +def schema_field(field_name, data_type, description, required=False): + return SchemaField( + field_name, + data_type, + description=description, + mode="REQUIRED" if required else "NULLABLE", + ) + + +schema_fields = [ + schema_field("cluster_name", "STRING", "cluster name", required=True), + schema_field("cluster_id", "STRING", "UUID for the cluster", required=True), + schema_field("entry_uuid", "STRING", "entry UUID for the job row", required=True), + schema_field( + "job_db_uuid", "STRING", "job db index from the slurm database", required=True + ), + schema_field("job_id_raw", "INT64", "raw job id", required=True), + schema_field("job_id", "STRING", "job id", required=True), + schema_field("state", "STRING", "final job state", required=True), + schema_field("job_name", "STRING", "job name"), + schema_field("partition", "STRING", "job partition"), + schema_field("submit_time", "DATETIME", "job submit time"), + schema_field("start_time", "DATETIME", "job start time"), + schema_field("end_time", "DATETIME", "job end time"), + schema_field("elapsed_raw", "INT64", "STRING", "job run time in seconds"), + # schema_field("elapsed_time", "INTERVAL", "STRING", "job run time interval"), + schema_field("timelimit_raw", "STRING", "job timelimit in minutes"), + schema_field("timelimit", "STRING", "job timelimit"), + # schema_field("num_tasks", "INT64", "number of allocated tasks in job"), + schema_field("nodelist", "STRING", "names of nodes allocated to job"), + schema_field("user", "STRING", "user responsible for job"), + schema_field("uid", "INT64", "uid of job user"), + schema_field("group", "STRING", "group of job user"), + schema_field("gid", "INT64", "gid of job user"), + schema_field("wckey", "STRING", "job wckey"), + schema_field("qos", "STRING", "job qos"), + schema_field("comment", "STRING", "job comment"), + schema_field("admin_comment", "STRING", "job admin comment"), + # extra will be added in 23.02 + # schema_field("extra", "STRING", "job extra field"), + schema_field("exitcode", "STRING", "job exit code"), + schema_field("alloc_cpus", "INT64", "count of allocated CPUs"), + schema_field("alloc_nodes", "INT64", "number of nodes allocated to job"), + schema_field("alloc_tres", "STRING", "allocated trackable resources (TRES)"), + # schema_field("system_cpu", "INTERVAL", "cpu time used by parent processes"), + # schema_field("cpu_time", "INTERVAL", "CPU time used (elapsed * cpu count)"), + schema_field("cpu_time_raw", "INT64", "CPU time used (elapsed * cpu count)"), + # schema_field("ave_cpu", "INT64", "Average CPU time of all tasks in job"), + # schema_field( + # "tres_usage_tot", + # "STRING", + # "Tres total usage by all tasks in job", + # ), +] + + +slurm_field_map = { + "job_db_uuid": "DBIndex", + "job_id_raw": "JobIDRaw", + "job_id": "JobID", + "state": "State", + "job_name": "JobName", + "partition": "Partition", + "submit_time": "Submit", + "start_time": "Start", + "end_time": "End", + "elapsed_raw": "ElapsedRaw", + "elapsed_time": "Elapsed", + "timelimit_raw": "TimelimitRaw", + "timelimit": "Timelimit", + "num_tasks": "NTasks", + "nodelist": "Nodelist", + "user": "User", + "uid": "Uid", + "group": "Group", + "gid": "Gid", + "wckey": "Wckey", + "qos": "Qos", + "comment": "Comment", + "admin_comment": "AdminComment", + # "extra": "Extra", + "exit_code": "ExitCode", + "alloc_cpus": "AllocCPUs", + "alloc_nodes": "AllocNodes", + "alloc_tres": "AllocTres", + "system_cpu": "SystemCPU", + "cpu_time": "CPUTime", + "cpu_time_raw": "CPUTimeRaw", + "ave_cpu": "AveCPU", + "tres_usage_tot": "TresUsageInTot", +} + +# new field name is the key for job_schema. Used to lookup the datatype when +# creating the job rows +job_schema = {field.name: field for field in schema_fields} +# Order is important here, as that is how they are parsed from sacct output +Job = namedtuple("Job", job_schema.keys()) # type: ignore +# ... see https://github.com/python/mypy/issues/848 + +client = bq.Client( + project=lookup().cfg.project, + credentials=util.default_credentials(), + client_options=util.create_client_options(util.ApiEndpoint.BQ), +) +dataset_id = f"{lookup().cfg.slurm_cluster_name}_job_data" +dataset = bq.DatasetReference(project=lookup().project, dataset_id=dataset_id) +table = bq.Table( + bq.TableReference(dataset, f"jobs_{lookup().cfg.slurm_cluster_name}"), schema_fields +) + + +class JobInsertionFailed(Exception): + pass + + +def make_job_row(job): + job_row = { + field_name: converters[field.field_type](job[field_name]) + for field_name, field in job_schema.items() + if field_name in job + } + job_row["entry_uuid"] = uuid.uuid4().hex + job_row["cluster_id"] = lookup().cfg.cluster_id + job_row["cluster_name"] = lookup().cfg.slurm_cluster_name + return job_row + + +def load_slurm_jobs(start, end): + states = ",".join( + ( + "BOOT_FAIL", + "CANCELLED", + "COMPLETED", + "DEADLINE", + "FAILED", + "NODE_FAIL", + "OUT_OF_MEMORY", + "PREEMPTED", + "REQUEUED", + "REVOKED", + "TIMEOUT", + ) + ) + start_iso = start.isoformat(timespec="seconds") + end_iso = end.isoformat(timespec="seconds") + # slurm_fields and bq_fields will be in matching order + slurm_fields = ",".join(slurm_field_map.values()) + bq_fields = slurm_field_map.keys() + cmd = ( + f"{SACCT} --start {start_iso} --end {end_iso} -X -D --format={slurm_fields} " + f"--state={states} --parsable2 --noheader --allusers --duplicates" + ) + text = run(cmd).stdout.splitlines() + # zip pairs bq_fields with the value from sacct + jobs = [dict(zip(bq_fields, line.split("|"))) for line in text] + + # The job index cache allows us to avoid sending duplicate jobs. This avoids a race condition with updating the database. + with shelve.open(str(job_idx_cache_path), flag="r") as job_idx_cache: + job_rows = [ + make_job_row(job) + for job in jobs + if str(job["job_db_uuid"]) not in job_idx_cache + ] + return job_rows + + +def init_table(): + global dataset + global table + dataset = client.create_dataset(dataset, exists_ok=True) # type: ignore + table = client.create_table(table, exists_ok=True) + until_found = retry.Retry(predicate=retry.if_exception_type(exceptions.NotFound)) + table = client.get_table(table, retry=until_found) + # cannot add required fields to an existing schema + table.schema = schema_fields + table = client.update_table(table, ["schema"]) + + +def purge_job_idx_cache(): + purge_time = datetime.now() - timedelta(minutes=30) + with shelve.open(str(job_idx_cache_path), writeback=True) as cache: + to_delete = [] + for idx, stamp in cache.items(): + if stamp < purge_time: + to_delete.append(idx) + for idx in to_delete: + del cache[idx] + + +def bq_submit(jobs): + try: + result = client.insert_rows(table, jobs) + except exceptions.NotFound as e: + print(f"failed to upload job data, table not yet found: {e}") + raise e + except Exception as e: + print(f"failed to upload job data: {e}") + raise e + if result: + pprint(jobs) + pprint(result) + raise JobInsertionFailed("failed to upload job data to big query") + print(f"successfully loaded {len(jobs)} jobs") + + +def get_time_window(): + if not timestamp_file.is_file(): + timestamp_file.touch() + try: + timestamp = datetime.strptime( + timestamp_file.read_text().rstrip(), SLURM_TIME_FORMAT + ) + # time window will overlap the previous by 10 minutes. Duplicates will be filtered out by the job_idx_cache + start = timestamp - timedelta(minutes=10) + except ValueError: + # timestamp 1 is 1 second after the epoch; timestamp 0 is special for sacct + start = datetime.fromtimestamp(1) + # end is now() truncated to the last second + end = datetime.now().replace(microsecond=0) + return start, end + + +def write_timestamp(time): + timestamp_file.write_text(time.isoformat(timespec="seconds")) + + +def update_job_idx_cache(jobs, timestamp): + with shelve.open(str(job_idx_cache_path), writeback=True) as job_idx_cache: + for job in jobs: + job_idx = str(job["job_db_uuid"]) + job_idx_cache[job_idx] = timestamp + + +def main(): + if not lookup().cfg.enable_bigquery_load: + print("bigquery load is not currently enabled") + exit(0) + init_table() + + start, end = get_time_window() + jobs = load_slurm_jobs(start, end) + # on failure, an exception will cause the timestamp not to be rewritten. So + # it will try again next time. If some writes succeed, we don't currently + # have a way to not submit duplicates next time. + if jobs: + num_batches = (len(jobs) - 1) // BQ_ROW_BATCH_SIZE + 1 + print( + f"loading {num_batches} batches of BigQuery data in batches of size : {BQ_ROW_BATCH_SIZE}" + ) + for batch_indx, job_indx in enumerate(range(0, len(jobs), BQ_ROW_BATCH_SIZE)): + print(f"loading BigQuery data batch {batch_indx} of {num_batches}") + bq_submit(jobs[job_indx : job_indx + BQ_ROW_BATCH_SIZE]) + write_timestamp(end) + update_job_idx_cache(jobs, end) + + +parser = argparse.ArgumentParser(description="submit slurm job data to big query") +parser.add_argument( + "timestamp_file", + nargs="?", + action="store", + type=Path, + help="specify timestamp file for reading and writing the time window start. Precedence over TIMESTAMP_FILE env var.", +) + +purge_job_idx_cache() +if __name__ == "__main__": + args = parser.parse_args() + if args.timestamp_file: + timestamp_file = args.timestamp_file.resolve() + main() diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py new file mode 100644 index 0000000000..d4a4477f83 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py @@ -0,0 +1,196 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Implementation of message queue that mimics interface of GCP (PubSub)[https://cloud.google.com/pubsub] + +Messages are stored on controller state disk (to survive controller re-creation) with following layout: + +// +ā”œ- +| ā””- +ā””- .staging + ā””- + ā””- + +One message is one immutable file, that will be deleted after acknowledgement. +NOTE: Implementation assumes that both `` and `.staging/` are on the same disk device, +so it can rely on atomic "move / rename" operation. +""" +from typing import Any +import util +import json +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +import os +import uuid + +import logging +log = logging.getLogger() + + +@dataclass(frozen=True) +class Message: + id: str + created: datetime + data: Any + + def to_json(self) -> dict[str, str]: + return dict( + id=self.id, + created=self.created.isoformat(), + data=self.data) + + @classmethod + def from_json(cls, data: dict[str, str]) -> 'Message': + return cls( + id=data['id'], + created=datetime.fromisoformat(data['created']), + data=data['data']) + +class Topic: + """ + Acts as PubSub topic (https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics). + We can have multiple instances of + """ + def __init__(self, path: Path, staging: Path) -> None: + self._path = path + self._staging = staging + + def _gen_id(self, created: datetime) -> str: + ts = created.strftime("%Y_%m_%d-%H_%M_%S") + suf = str(uuid.uuid4())[:8] + return f"{ts}-{suf}" + + def publish(self, data: Any) -> None: + created = util.now() + id = self._gen_id(created) + msg = Message(id=id, created=created, data=data) + + staged = self._staging / msg.id + dst = self._path / msg.id + + # Write to stagin area first then perform atomic move + # to prevent "reads of partial writes" + staged.write_text(json.dumps(msg.to_json())) + util.chown_slurm(staged) + staged.rename(dst) + + +class Subscription: + """ + Acts as PubSub subscription (https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions) + with following settings: + + ``` + ackDeadlineSeconds = +Inf # don't resend message that was already being delivered but not acked yet + retainAckedMessages = False # don't persist messages that were already acked + enableMessageOrdering = True # delivers messages in chronoligical order + messageRetentionDuration = +Inf # don't expire messages + deadLetterPolicy = None # "deadlettering" is disabled, subscriber should take care of any poisonous messages + retryPolicy = { # NACKed message will be re-delievered after some time + minimumBackoff = 30s # NOTE: Practically there is no timer, but Subscription instance will not try to re-deliver NACKed messages. + maximumBackoff = 30s # Assumes that slurmsync runs every 30+ sec. + } + ``` + + IMPORTANT: Should only be run as part of slurmsync, + this is our way to ensure that at most one instance exists at a time. + There is no concurancy safeguards in place, avoid multithreaded `pull`, + while multithreaded `ack` & `modify_ack_deadline` are OK. + """ + + def __init__(self, path: Path) -> None: + self._path: Path = path + # contains ALL messages pulled by this subscription instance + # both acked, nacked, and still being processed + # used to prevent double delivery within lifetime of subscription (slurmsync) + self._pulled: set[str] = set() + + def _delete(self, id: str) -> None: + log.debug(f"removing {id}") + try: + os.unlink(self._path / id) + except: + log.exception(f"Failed to remove message {id}") + + def _read_msg(self, id: str) -> Message | None: + try: + with open(self._path / id, 'r') as f: + content = json.loads(f.read()) + return Message.from_json(content) + except Exception: + log.exception(f"Failed to read message {id}") + self._delete(id) # delete message to reduce "deadlettering" + return None + + def pull(self, max_messages: int) -> list[Message]: + if not self._path.exists(): + log.warning(f"Topic {self._path} does not exist") + return [] + res = [] + ls = sorted(os.listdir(self._path)) + for name in ls: + msg = self._read_msg(name) + if msg is not None and msg.id not in self._pulled: + self._pulled.add(msg.id) + res.append(msg) + + if len(res) >= max_messages: + break + return res + + + def ack(self, ids: list[str]) -> None: + for id in ids: + self._delete(id) + + + def modify_ack_deadline(self, ids: list[str], deadline: int) -> None: + """ + Modifies the ack deadline for a specific message. + IMPORTANT: Only accepts deadline=0, which is a way to NACK + Any other values are also meaningless due to ackDeadlineSeconds==+Inf + """ + assert deadline == 0 # no op, next subscriber (slurmsync) will pick this up + + +# Topics and Subscriptions are singletons +# TODO: consider making thread-safe +_topics = {} +_subscriptions = {} + +def _make_path(name: str) -> Path: + p = util.slurmdirs.state / "pubsub" / name + p.mkdir(parents=True, exist_ok=True) + util.chown_slurm(p) + return p + +def _make_staging_path(name: str) -> Path: + p = util.slurmdirs.state / "pubsub" / ".staging" / name + p.mkdir(parents=True, exist_ok=True) + util.chown_slurm(p) + return p + +def topic(name: str) -> Topic: + if name not in _topics: + _topics[name] = Topic(_make_path(name), _make_staging_path(name)) + return _topics[name] + +def subscription(name: str) -> Subscription: + if name not in _subscriptions: + _subscriptions[name] = Subscription(_make_path(name)) + return _subscriptions[name] diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py new file mode 100644 index 0000000000..35eacc05e0 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py @@ -0,0 +1,214 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Optional + +import util +import uuid +from addict import Dict as NSDict # type: ignore +from datetime import timedelta +from collections import defaultdict +import logging +from time import sleep + +log = logging.getLogger() + + +def _duration(flex_options: NSDict, job_id: Optional[int], lkp: util.Lookup) -> int: + dur = flex_options.max_run_duration + if not job_id or not flex_options.use_job_duration: + return dur + + job = lkp.job(job_id) + if not job or not job.duration: + return dur + + if timedelta(minutes=10) <= job.duration <= timedelta(weeks=1): + return int(job.duration.total_seconds()) + + log.info("Job TimeLimit cannot be less than 10 minutes or exceed one week") + return dur + + +def resume_flex_chunk(nodes: List[str], job_id: Optional[int], lkp: util.Lookup) -> None: + assert nodes + model = nodes[0] + nodeset = lkp.node_nodeset(model) + zones = nodeset.zone_policy_allow + assert len(zones) == 1 + zone = zones[0] + + assert nodeset.dws_flex.enabled + + uid = str(uuid.uuid4())[:8] + if job_id: + mig_name = f"{lkp.cfg.slurm_cluster_name}-{nodeset.nodeset_name}-job-{job_id}-{uid}" + else: + mig_name = f"{lkp.cfg.slurm_cluster_name}-{nodeset.nodeset_name}-{uid}" + + # Create MIG + req = lkp.compute.instanceGroupManagers().insert( + project=lkp.project, + zone=zone, + body=dict( + name=mig_name, + versions=[dict( + instanceTemplate=nodeset.instance_template)], + targetSize=0, + # TODO(FLEX): uncomment once moved to RMIG + # distributionPolicy=dict( + # zones=[ + # dict(zone=f"zones/{z}") for z in nodeset.zone_policy_allow + # ], + # targetShape="ANY_SINGLE_ZONE" ), + #updatePolicy = dict( + # instanceRedistributionType = "NONE" ), + instanceLifecyclePolicy=dict( + defaultActionOnFailure= "DO_NOTHING" ), # TODO(FLEX): Not supported yet, migrate once supported + ) + ) + util.log_api_request(req) + op = req.execute() + res = util.wait_for_operation(op) + assert "error" not in res, f"{res}" + + # Create resize request + req = lkp.compute.instanceGroupManagerResizeRequests().insert( + project=lkp.project, + zone=zone, + instanceGroupManager=mig_name, + body=dict( + name="initial-resize", + instances=[dict(name=n) for n in nodes], + requested_run_duration=dict( + seconds=_duration(nodeset.dws_flex, job_id, lkp) + ) + ) + ) + util.log_api_request(req) + op = req.execute() + res = util.wait_for_operation(op) + assert "error" not in res, f"{res}" + +def _suspend_flex_mig(mig_self_link: str, nodes: List[str], lkp: util.Lookup) -> None: + assert nodes + model = nodes[0] + nodeset = lkp.node_nodeset(model) + zones = nodeset.zone_policy_allow + assert len(zones) == 1 + zone = zones[0] + project=lkp.project + instanceGroupManager=util.trim_self_link(mig_self_link) + + links = [ + f"zones/{inst.zone}/instances/{inst.name}" + for inst in [ + lkp.instance(node) for node in nodes + ] if inst + ] + + target_mig=lkp.get_mig(lkp.project, zone, instanceGroupManager) + assert target_mig + + # TODO(FLEX): This will not work if MIG didn't obtain capacity yet. + # The request will fail and MIG will continue provisioning. + # Instead whole MIG should be deleted. + # + All other instances in MIG are not provisioned also, safe to delete + # - Need to come up will clear test to differentiate non-provisioned MIG and single VM being down; + # Particularly CRITICAL due to ActionOnFailure=DO_NOTHING + # - Need to `down_nodes_notify_jobs` for all nodes in MIG, make sure that it doesn't interfere with Slurm suspend-flow. + + if target_mig["targetSize"] == len(nodes): #We can just delete the whole MIG in this case + req = lkp.compute.instanceGroupManagers().delete( + project=project, + zone=zone, + instanceGroupManager=instanceGroupManager, + ) + else: + req = lkp.compute.instanceGroupManagers().deleteInstances( + project=project, + zone=zone, + instanceGroupManager=instanceGroupManager, + body=dict( + instances=links, + skipInstancesOnValidationError=True, + ) + ) + + util.log_api_request(req) + op = req.execute() + + res = util.wait_for_operation(op) + assert "error" not in res, f"{res}" + +def _suspend_provisioning_inst(nodes:List[str], node_template:str, lkp: util.Lookup) -> None: + assert nodes + model = nodes[0] + nodeset = lkp.node_nodeset(model) + zones = nodeset.zone_policy_allow + assert len(zones) == 1 + zone = zones[0] + + mig_list=lkp.get_mig_list(lkp.project, zone) #Validated via terraform that this is one + + # FLEX (#TODO): If we enter this conditional it's likely this was called so early that MIG creation hasn't started + # Consider potentially retrying? No natural mechanism for retry currently but we could + # perhaps use slurmsync and then try it again to ensure it wasn't a case of being too early. + # This is important since we're now enabling long ResumeTimeout (Slurm won't call suspend on node within reasonable timeframe) + # so until we do this is slurmsync this is a temporary workaround. + + if not mig_list or not mig_list.get("items"): + log.info("No matching MIG found to delete!") + sleep(5) + mig_list=lkp.get_mig_list(lkp.project, zone) + if not mig_list or not mig_list.get("items"): + return + + for mig in mig_list["items"]: + if mig["instanceTemplate"] == node_template: + if mig["currentActions"]["creating"] > 0 and mig["targetSize"] == mig["currentActions"]["creating"]: + req = lkp.compute.instanceGroupManagers().delete( + project=lkp.project, + zone=zone, + instanceGroupManager=util.trim_self_link(mig["selfLink"]), + ) + + util.log_api_request(req) + op = req.execute() + + res = util.wait_for_operation(op) + assert "error" not in res, f"{res}" + return + + log.info("No matching MIG found to delete!") + +def suspend_flex_nodes(nodes: List[str], lkp: util.Lookup) -> None: + by_mig = defaultdict(list) + not_provisioned = defaultdict(list) + for node in nodes: + inst = lkp.instance(node) + if not inst: + not_provisioned[lkp.node_template(node)].append(node) + else: + mig = inst.metadata.get("created-by") + if not mig: + log.error(f"Can not suspend {node}, can not find associated MIG") + continue + by_mig[mig].append(node) + + for mig, nodes in by_mig.items(): + _suspend_flex_mig(mig, nodes, lkp) + + for node_template, nodes in not_provisioned.items(): + _suspend_provisioning_inst(nodes, node_template, lkp) diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt new file mode 100644 index 0000000000..2ab3162ccf --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt @@ -0,0 +1,9 @@ +pytest +pytest-mock +pytest_unordered +mock + +types-mock +types-httplib2 +types-requests +types-PyYAML diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt new file mode 100644 index 0000000000..d00e0cf3a3 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt @@ -0,0 +1,18 @@ +addict==2.4.0 +google-api-core==2.19.0 +google-api-python-client==2.93.0 +google-auth==2.22.0 +google-auth-httplib2==0.1.0 +google-cloud-bigquery==3.11.3 +google-cloud-core==2.3.3 +google-cloud-secret-manager~=2.22 +google-cloud-storage==2.10.0 +google-cloud-tpu==1.10.0 +google-resumable-media==2.5.0 +googleapis-common-protos==1.59.1 +grpcio==1.56.2 +grpcio-status==1.56.2 +httplib2==0.22.0 +more-executors==2.11.4 +pyyaml==6.0.2 +requests==2.32.4 diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py new file mode 100644 index 0000000000..966a6bb852 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py @@ -0,0 +1,664 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Optional, Dict, Any +import argparse +from datetime import timedelta +import shlex +import json +import logging +import os +import yaml +import collections +from pathlib import Path +from dataclasses import dataclass +from addict import Dict as NSDict # type: ignore + +import util +from util import ( + chunked, + ensure_execute, + execute_with_futures, + log_api_request, + map_with_futures, + run, + separate, + to_hostlist, + trim_self_link, + wait_for_operation, +) +from util import lookup, ReservationDetails +import tpu +import mig_flex + +log = logging.getLogger() + +PLACEMENT_MAX_CNT = 1500 +# Placement group needs to be the same for an entire bulk_insert hence +# if placement is used the actual BULK_INSERT_LIMIT will be +# max([1000, PLACEMENT_MAX_CNT]) +BULK_INSERT_LIMIT = 5000 + +# https://cloud.google.com/compute/docs/instance-groups#types_of_managed_instance_groups +ZONAL_MIG_SIZE_LIMIT = 1000 + + +@dataclass(frozen=True) +class ResumeJobData: + job_id: int + partition: str + nodes_alloc: List[str] + +@dataclass(frozen=True) +class ResumeData: + jobs: List[ResumeJobData] + + +def get_resume_file_data() -> Optional[ResumeData]: + if not (path := os.getenv("SLURM_RESUME_FILE")): + log.error("SLURM_RESUME_FILE was not in environment. Cannot get detailed job, node, partition allocation data.") + return None + blob = Path(path).read_text() + log.debug(f"Resume data: {blob}") + data = json.loads(blob) + + jobs = [] + for jo in data.get("jobs", []): + job = ResumeJobData( + job_id = jo.get("job_id"), + partition = jo.get("partition"), + nodes_alloc = util.to_hostnames(jo.get("nodes_alloc")), + ) + jobs.append(job) + return ResumeData(jobs=jobs) + +def instance_properties(nodeset: NSDict, model:str, placement_group:Optional[str], labels:Optional[dict], job_id:Optional[int]): + props = NSDict() + + if labels: # merge in extra labels on instance and disks + template_link = lookup().node_template(model) + template_info = lookup().template_info(template_link) + + props.labels = {**template_info.labels, **labels} + + for disk in template_info.disks: + if disk.initializeParams.get("diskType", "local-ssd") == "local-ssd": + continue # do not label local ssd + disk.initializeParams.labels.update(labels) + props.disks = template_info.disks + + if placement_group: + props.resourcePolicies = [placement_group] + + if reservation := lookup().nodeset_reservation(nodeset): + update_reservation_props(reservation, props, placement_group, reservation.calendar) + + if (fr := lookup().future_reservation(nodeset)) and fr.specific: + assert fr.active_reservation + update_reservation_props(fr.active_reservation, props, placement_group, fr.calendar) + + if props.resourcePolicies: + props.scheduling.onHostMaintenance = "TERMINATE" + + if nodeset.maintenance_interval: + props.scheduling.maintenanceInterval = nodeset.maintenance_interval + + if nodeset.dws_flex.enabled and nodeset.dws_flex.use_bulk_insert: + update_props_dws(props, nodeset.dws_flex, job_id) + + # Override with properties explicit specified in the nodeset + props.update(nodeset.get("instance_properties") or {}) + return props + +def update_reservation_props(reservation:ReservationDetails, props:NSDict, placement_group:Optional[str], calendar_mode:bool) -> None: + props.reservationAffinity = { + "consumeReservationType": "SPECIFIC_RESERVATION", + "key": f"compute.{util.universe_domain()}/reservation-name", + "values": [reservation.bulk_insert_name], + } + + if reservation.dense or calendar_mode: + props.scheduling.provisioningModel = "RESERVATION_BOUND" + + # Figure out `resourcePolicies` + if reservation.policies: # use ones already attached to reservations + props.resourcePolicies = reservation.policies + elif reservation.dense and placement_group: # use once created by Slurm + props.resourcePolicies = [placement_group] + else: # vanilla reservations don't support external policies + props.resourcePolicies = [] + log.info( + f"reservation {reservation.bulk_insert_name} is being used with resourcePolicies: {props.resourcePolicies}") + +def update_props_dws(props: NSDict, dws_flex: NSDict, job_id: Optional[int]) -> None: + props.scheduling.onHostMaintenance = "TERMINATE" + props.scheduling.instanceTerminationAction = "DELETE" + props.reservationAffinity['consumeReservationType'] = "NO_RESERVATION" + props.scheduling.maxRunDuration['seconds'] = dws_flex_duration(dws_flex, job_id) + +def dws_flex_duration(dws_flex: NSDict, job_id: Optional[int]) -> int: + max_duration = dws_flex.max_run_duration + if dws_flex.use_job_duration and job_id is not None and (job := lookup().job(job_id)) and job.duration: + if timedelta(seconds=30) <= job.duration <= timedelta(weeks=1): + max_duration = int(job.duration.total_seconds()) + else: + log.info("Job TimeLimit cannot be less than 30 seconds or exceed one week") + return max_duration + +def create_instances_request(nodes: List[str], placement_group: Optional[str], excl_job_id: Optional[int]): + """Call regionInstances.bulkInsert to create instances""" + assert 0 < len(nodes) <= BULK_INSERT_LIMIT + + # model here indicates any node that can be used to describe the rest + model = next(iter(nodes)) + log.debug(f"create_instances_request: {model} placement: {placement_group}") + + nodeset = lookup().node_nodeset(model) + template = lookup().node_template(model) + labels = {"slurm_job_id": excl_job_id} if excl_job_id else None + + body = dict( + count = len(nodes), + sourceInstanceTemplate = template, + # key is instance name, value overwrites properties (no overwrites) + perInstanceProperties = {k: {} for k in nodes}, + instanceProperties = instance_properties( + nodeset, model, placement_group, labels, excl_job_id + ), + ) + + if placement_group and excl_job_id is not None: + pass # do not set minCount to force "all or nothing" behavior + else: + body["minCount"] = 1 + + zone_allow = nodeset.zone_policy_allow or [] + zone_deny = nodeset.zone_policy_deny or [] + + if len(zone_allow) == 1: # if only one zone is used, use zonal BulkInsert API, as less prone to errors + api_method = lookup().compute.instances().bulkInsert + method_args = {"zone": zone_allow[0]} + else: + api_method = lookup().compute.regionInstances().bulkInsert + method_args = {"region": lookup().node_region(model)} + + body["locationPolicy"] = dict( + locations = { + **{ f"zones/{z}": {"preference": "ALLOW"} for z in zone_allow }, + **{ f"zones/{z}": {"preference": "DENY"} for z in zone_deny }}, + targetShape = nodeset.zone_target_shape, + ) + + req = api_method( + project=lookup().project, + body=body, + **method_args) + log.debug(f"new request: endpoint={req.methodId} nodes={to_hostlist(nodes)}") + log_api_request(req) + return req + +@dataclass() +class PlacementAndNodes: + placement: Optional[str] + nodes: List[str] + +@dataclass(frozen=True) +class BulkChunk: + nodes: List[str] + prefix: str # - + chunk_idx: int + excl_job_id: Optional[int] + placement_group: Optional[str] = None + + @property + def name(self): + if self.placement_group is not None: + return f"{self.prefix}:job{self.excl_job_id}:{self.placement_group}:{self.chunk_idx}" + if self.excl_job_id is not None: + return f"{self.prefix}:job{self.excl_job_id}:{self.chunk_idx}" + return f"{self.prefix}:{self.chunk_idx}" + + +def group_nodes_bulk(nodes: List[str], resume_data: Optional[ResumeData], lkp: util.Lookup): + """group nodes by nodeset, placement_group, exclusive_job_id if any""" + if resume_data is None: # all nodes will be considered jobless + resume_data = ResumeData(jobs=[]) + + nodes_set = set(nodes) # turn into set to simplify intersection + non_excl = nodes_set.copy() + groups : Dict[Optional[int], List[PlacementAndNodes]] = {} # excl_job_id|none -> PlacementAndNodes + + # expand all exclusive job nodelists + for job in resume_data.jobs: + if not lkp.cfg.partitions[job.partition].enable_job_exclusive: + continue + + groups[job.job_id] = [] + # placement group assignment is based on all allocated nodes, ... + for pn in create_placements(job.nodes_alloc, job.job_id, lkp): + groups[job.job_id].append( + PlacementAndNodes( + placement=pn.placement, + #... but we only want to handle nodes in nodes_resume in this run. + nodes = sorted(set(pn.nodes) & nodes_set) + )) + non_excl.difference_update(job.nodes_alloc) + + groups[None] = create_placements(sorted(non_excl), excl_job_id=None, lkp=lkp) + + def chunk_nodes(nodes: List[str]): + if not nodes: + return [] + + model = nodes[0] + + if lkp.is_flex_node(model): + chunk_size = ZONAL_MIG_SIZE_LIMIT + elif lkp.node_is_tpu(model): + ns_name = lkp.node_nodeset_name(model) + chunk_size = tpu.TPU.make(ns_name, lkp).vmcount + else: + chunk_size = BULK_INSERT_LIMIT + + return chunked(nodes, n=chunk_size) + + chunks = [ + BulkChunk( + nodes=nodes_chunk, + prefix=lkp.node_prefix(nodes_chunk[0]), # - + excl_job_id = job_id, + placement_group=pn.placement, + chunk_idx=i) + + for job_id, placements in groups.items() + for pn in placements if pn.nodes + for i, nodes_chunk in enumerate(chunk_nodes(pn.nodes)) + ] + return {chunk.name: chunk for chunk in chunks} + + +def resume_nodes(nodes: List[str], resume_data: Optional[ResumeData]): + """resume nodes in nodelist""" + lkp = lookup() + # Prevent dormant nodes associated with a future reservation from being resumed + nodes, dormant_fr_nodes = util.separate(lkp.is_dormant_fr_node, nodes) + + if dormant_fr_nodes: + log.warning(f"Resume was unable to resume future reservation nodes={dormant_fr_nodes}") + down_nodes_notify_jobs(dormant_fr_nodes, "Reservation is not active, nodes cannot be resumed", resume_data) + + nodes, flex_managed = util.separate(lkp.is_provisioning_flex_node, nodes) + if flex_managed: + log.warning(f"Resume was unable to resume nodes={flex_managed} already managed by MIGs") + down_nodes_notify_jobs(flex_managed, "VM is managed MIG, can not be resumed", resume_data) + + if not nodes: + log.info("No nodes to resume") + return + + nodes = sorted(nodes, key=lkp.node_prefix) + grouped_nodes = group_nodes_bulk(nodes, resume_data, lkp) + + if log.isEnabledFor(logging.DEBUG): + grouped_nodelists = { + group: to_hostlist(chunk.nodes) for group, chunk in grouped_nodes.items() + } + log.debug( + "node bulk groups: \n{}".format(yaml.safe_dump(grouped_nodelists).rstrip()) + ) + + tpu_chunks, flex_chunks = [], [] + bi_inserts = {} + + for group, chunk in grouped_nodes.items(): + model = chunk.nodes[0] + + if lkp.node_is_tpu(model): + tpu_chunks.append(chunk.nodes) + elif lkp.is_flex_node(model): + flex_chunks.append(chunk) + else: + bi_inserts[group] = create_instances_request( + chunk.nodes, chunk.placement_group, chunk.excl_job_id + ) + + for chunk in flex_chunks: + mig_flex.resume_flex_chunk(chunk.nodes, chunk.excl_job_id, lkp) + + # execute all bulkInsert requests with batch + bulk_ops = dict( + zip(bi_inserts.keys(), map_with_futures(ensure_execute, bi_inserts.values())) + ) + log.debug(f"bulk_ops={yaml.safe_dump(bulk_ops)}") + started = { + group: op for group, op in bulk_ops.items() if not isinstance(op, Exception) + } + failed = { + group: err for group, err in bulk_ops.items() if isinstance(err, Exception) + } + if failed: + failed_reqs = [str(e) for e in failed.items()] + log.error("bulkInsert API failures: {}".format("; ".join(failed_reqs))) + for ident, exc in failed.items(): + down_nodes_notify_jobs(grouped_nodes[ident].nodes, f"GCP Error: {exc._get_reason()}", resume_data) # type: ignore + + if log.isEnabledFor(logging.DEBUG): + for group, op in started.items(): + group_nodes = grouped_nodelists[group] + name = op["name"] + gid = op["operationGroupId"] + log.debug( + f"new bulkInsert operation started: group={group} nodes={group_nodes} name={name} operationGroupId={gid}" + ) + # wait for all bulkInserts to complete and log any errors + bulk_operations = {group: wait_for_operation(op) for group, op in started.items()} + + # Start TPU after regular nodes so that regular nodes are not affected by the slower TPU nodes + execute_with_futures(tpu.start_tpu, tpu_chunks) + + for group, op in bulk_operations.items(): + _handle_bulk_insert_op(op, grouped_nodes[group].nodes, resume_data) + + +def _get_failed_zonal_instance_inserts(bulk_op: Any, zone: str, lkp: util.Lookup) -> list[Any]: + group_id = bulk_op["operationGroupId"] + user = bulk_op["user"] + started = bulk_op["startTime"] + ended = bulk_op["endTime"] + + fltr = f'(user eq "{user}") AND (operationType eq "insert") AND (creationTimestamp > "{started}") AND (creationTimestamp < "{ended}")' + act = lkp.compute.zoneOperations() + req = act.list(project=lkp.project, zone=zone, filter=fltr) + ops = [] + while req is not None: + result = util.ensure_execute(req) + for op in result.get("items", []): + if op.get("operationGroupId") == group_id and "error" in op: + ops.append(op) + req = act.list_next(req, result) + return ops + + +def _get_failed_instance_inserts(bulk_op: Any, lkp: util.Lookup) -> list[Any]: + zones = set() # gather zones that had failed inserts + for loc, stat in bulk_op.get("instancesBulkInsertOperationMetadata", {}).get("perLocationStatus", {}).items(): + pref, zone = loc.split("/", 1) + if not pref == "zones": + log.error(f"Unexpected location: {loc} in operation {bulk_op['name']}") + continue + if stat.get("targetVmCount", 0) != stat.get("createdVmCount", 0): + zones.add(zone) + + res = [] + for zone in zones: + res.extend(_get_failed_zonal_instance_inserts(bulk_op, zone, lkp)) + return res + +def _handle_bulk_insert_op(op: Dict, nodes: List[str], resume_data: Optional[ResumeData]) -> None: + """ + Handles **DONE** BulkInsert operations + """ + assert op["operationType"] == "bulkInsert" and op["status"] == "DONE", f"unexpected op: {op}" + + group_id = op["operationGroupId"] + if "error" in op: + error = op["error"]["errors"][0] + log.error( + f"bulkInsert operation error: {error['code']} name={op['name']} operationGroupId={group_id} nodes={to_hostlist(nodes)}" + ) + + created = 0 + for status in op["instancesBulkInsertOperationMetadata"]["perLocationStatus"].values(): + created += status.get("createdVmCount", 0) + if created == len(nodes): + log.info(f"created {len(nodes)} instances: nodes={to_hostlist(nodes)}") + return # no need to gather status of insert-operations. + + # TODO: don't gather insert-operations per bulkInsert request, instead aggregate it + # across all bulkInserts (goes one level above this function) + failed = _get_failed_instance_inserts(op, util.lookup()) + + # Multiple errors are possible, group by all of them (joined string codes) + by_error_inserts = util.groupby_unsorted( + failed, + lambda op: "+".join(err["code"] for err in op["error"]["errors"]), + ) + for code, failed_ops in by_error_inserts: + failed_ops = list(failed_ops) + failed_nodes = [trim_self_link(op["targetLink"]) for op in failed_ops] + hostlist = util.to_hostlist(failed_nodes) + log.error( + f"{len(failed_nodes)} instances failed to start: {code} ({hostlist}) operationGroupId={group_id}" + ) + + msg = "; ".join( + f"{err['code']}: {err['message'] if 'message' in err else 'no message'}" + for err in failed_ops[0]["error"]["errors"] + ) + if code != "RESOURCE_ALREADY_EXISTS": + down_nodes_notify_jobs(failed_nodes, f"GCP Error: {msg}", resume_data) + log.error( + f"errors from insert for node '{failed_nodes[0]}' ({failed_ops[0]['name']}): {msg}" + ) + + +def down_nodes_notify_jobs(nodes: List[str], reason: str, resume_data: Optional[ResumeData]) -> None: + """set nodes down with reason""" + nodes_set = set(nodes) # turn into set to speed up intersection + jobs = resume_data.jobs if resume_data else [] + reason_quoted = shlex.quote(reason) + + for job in jobs: + if not (set(job.nodes_alloc) & nodes_set): + continue + run(f"{lookup().scontrol} update jobid={job.job_id} admincomment={reason_quoted}", check=False) + run(f"{lookup().scontrol} notify {job.job_id} {reason_quoted}", check=False) + + nodelist = util.to_hostlist(nodes) + log.error(f"Marking nodes {nodelist} as DOWN, reason: {reason}") + run(f"{lookup().scontrol} update nodename={nodelist} state=down reason={reason_quoted}", check=False) + + + + +def create_placement_request(pg_name: str, region: str, max_distance: Optional[int]): + config = { + "name": pg_name, + "region": region, + "groupPlacementPolicy": { + "collocation": "COLLOCATED", + "maxDistance": max_distance + }, + } + + request = lookup().compute.resourcePolicies().insert( + project=lookup().project, region=region, body=config + ) + log_api_request(request) + return request + + +def create_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: + nodeset_map = collections.defaultdict(list) + for node in nodes: # split nodes on nodesets + nodeset_map[lkp.node_nodeset_name(node)].append(node) + + placements = [] + for _, ns_nodes in nodeset_map.items(): + placements.extend(create_nodeset_placements(ns_nodes, excl_job_id, lkp)) + return placements + + +def _allocate_nodes_to_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: + # canned result for no placement policies created + no_pp = [PlacementAndNodes(placement=None, nodes=nodes)] + + if excl_job_id and len(nodes) < 2: + return no_pp # don't create placement_policy for just one node + + model = nodes[0] + nodeset = lkp.node_nodeset(model) + + if lkp.is_flex_node(model): + return no_pp # TODO(FLEX): Add support for workload policies + if lkp.node_is_tpu(model): + return no_pp + if not (nodeset.enable_placement and valid_placement_node(model)): + return no_pp + + max_count = calculate_chunk_size(nodeset, lkp) + + name_prefix = f"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-{nodeset.nodeset_name}" + if excl_job_id: # simply chunk given nodes by max size of placement + return [ + PlacementAndNodes(placement=f"{name_prefix}-{excl_job_id}-{i}", nodes=chunk) + for i, chunk in enumerate(chunked(nodes, n=max_count)) + ] + + # split whole nodeset (not only nodes to resume) into chunks of max size of placement + # create placements (most likely already exists) placements for requested nodes + chunks = collections.defaultdict(list) # chunk_id -> nodes + invalid = [] + + for node in nodes: + try: + chunk = lkp.node_index(node) // max_count + chunks[chunk].append(node) + except: + invalid.append(node) + + placements = [ + # NOTE: use 0 instead of job_id for consistency with previous SlurmGCP behavior + PlacementAndNodes(placement=f"{name_prefix}-0-{c_id}", nodes=c_nodes) + for c_id, c_nodes in chunks.items() + ] + + if invalid: + placements.append(PlacementAndNodes(placement=None, nodes=invalid)) + log.error(f"Could not find placement for nodes with unexpected names: {to_hostlist(invalid)}") + + return placements + +def calculate_chunk_size(nodeset: NSDict, lkp: util.Lookup) -> int: + # Calculates the chunk size based on max distance value received + machine_type = lkp.template_info(nodeset.instance_template).machine_type.family + max_distance = nodeset.placement_max_distance + if max_distance == 1: + return 22 + elif max_distance == 2: + if machine_type.startswith("a3"): + return 256 + else: + return 150 + elif max_distance == 3: + return 1500 + else: + return PLACEMENT_MAX_CNT + +def create_nodeset_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: + placements = _allocate_nodes_to_placements(nodes, excl_job_id, lkp) + region = lkp.node_region(nodes[0]) + max_distance = lkp.node_nodeset(nodes[0]).get('placement_max_distance') + + if log.isEnabledFor(logging.DEBUG): + debug_p = {p.placement: to_hostlist(p.nodes) for p in placements} + log.debug( + f"creating {len(placements)} placement groups: \n{yaml.safe_dump(debug_p).rstrip()}" + ) + + requests = { + p.placement: create_placement_request(p.placement, region, max_distance) for p in placements if p.placement + } + if not requests: + return placements + # TODO: aggregate all requests for whole resume and execute them at once (don't limit to nodeset/job) + ops = dict( + zip(requests.keys(), map_with_futures(ensure_execute, requests.values())) + ) + + def classify_result(item): + op = item[1] + if not isinstance(op, Exception): + return "submitted" + if all(e.get("reason") == "alreadyExists" for e in op.error_details): # type: ignore + return "redundant" + return "failed" + + grouped_ops = dict(util.groupby_unsorted(list(ops.items()), classify_result)) + submitted, redundant, failed = ( + dict(grouped_ops.get(key, {})) for key in ("submitted", "redundant", "failed") + ) + if redundant: + log.warning( + "placement policies already exist: {}".format(",".join(redundant.keys())) + ) + if failed: + reqs = [f"{e}" for _, e in failed.values()] + log.fatal("failed to create placement policies: {}".format("; ".join(reqs))) + operations = {group: wait_for_operation(op) for group, op in submitted.items()} + for group, op in operations.items(): + if "error" in op: + msg = "; ".join( + f"{err['code']}: {err['message'] if 'message' in err else 'no message'}" + for err in op["error"]["errors"] + ) + log.error( + f"placement group failed to create: '{group}' ({op['name']}): {msg}" + ) + + log.info( + f"created {len(operations)} placement groups ({to_hostlist(operations.keys())})" + ) + return placements + + +def valid_placement_node(node: str) -> bool: + invalid_types = frozenset(["e2", "t2d", "n1", "t2a", "m1", "m2", "m3"]) + mt = lookup().node_template_info(node).machineType + if mt.split("-")[0] in invalid_types: + log.warn(f"Unsupported machine type for placement policy: {mt}.") + log.warn( + f"Please do not use any the following machine types with placement policy: ({','.join(invalid_types)})" + ) + return False + return True + + +def main(nodelist: str) -> None: + """main called when run as script""" + log.debug(f"ResumeProgram {nodelist}") + # Filter out nodes not in config.yaml + other_nodes, nodes = separate( + lookup().is_power_managed_node, util.to_hostnames(nodelist) + ) + if other_nodes: + log.error( + f"Ignoring non-power-managed nodes '{to_hostlist(other_nodes)}' from '{nodelist}'" + ) + + if not nodes: + log.info("No nodes to resume") + return + resume_data = get_resume_file_data() + log.info(f"resume {util.to_hostlist(nodes)}") + resume_nodes(nodes, resume_data) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("nodelist", help="list of nodes to resume") + args = util.init_log_and_parse(parser) + main(args.nodelist) diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh new file mode 100644 index 0000000000..a180761914 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +PYTHON_SCRIPT="${SCRIPT_DIR}/resume.py" + +# Capture all arguments passed by Slurm (the nodelist). +ALL_ARGS=("$@") + +# This array will hold extra argument for resume.py, like the resume data file. +UNIQUE_RESUME_FILE="" + +# Handle SLURM_RESUME_FILE if provided +if [ -n "${SLURM_RESUME_FILE-}" ] && [ -f "$SLURM_RESUME_FILE" ]; then + SAFE_DIR="/tmp/slurm_resume_data" + mkdir -p "$SAFE_DIR" + + UNIQUE_RESUME_FILE="${SAFE_DIR}/resumedata.$$.json" + cp "$SLURM_RESUME_FILE" "$UNIQUE_RESUME_FILE" +fi + +SLURM_RESUME_FILE="${UNIQUE_RESUME_FILE}" +"${PYTHON_SCRIPT}" "${ALL_ARGS[@]}" & +disown + +exit 0 diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py new file mode 100644 index 0000000000..0d27e6cfac --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py @@ -0,0 +1,648 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import os +import shutil +import subprocess +import stat +import time +import yaml +from pathlib import Path +import functools + +import util +from util import ( + lookup, + dirs, + slurmdirs, + run, + install_custom_scripts, +) +import conf +import slurmsync + +from setup_network_storage import ( + setup_network_storage, + setup_nfs_exports, +) + + +log = logging.getLogger() + + +MOTD_HEADER = """ + SSSSSSS + SSSSSSSSS + SSSSSSSSS + SSSSSSSSS + SSSS SSSSSSS SSSS + SSSSSS SSSSSS + SSSSSS SSSSSSS SSSSSS + SSSS SSSSSSSSS SSSS + SSS SSSSSSSSS SSS + SSSSS SSSS SSSSSSSSS SSSS SSSSS + SSS SSSSSS SSSSSSSSS SSSSSS SSS + SSSSSS SSSSSSS SSSSSS + SSS SSSSSS SSSSSS SSS + SSSSS SSSS SSSSSSS SSSS SSSSS + S SSS SSSSSSSSS SSS S + SSS SSSS SSSSSSSSS SSSS SSS + S SSS SSSSSS SSSSSSSSS SSSSSS SSS S + SSSSS SSSSSS SSSSSSSSS SSSSSS SSSSS + S SSSSS SSSS SSSSSSS SSSS SSSSS S + S SSS SSS SSS SSS S + S S S S + SSS + SSS + SSS + SSS + SSSSSSSSSSSS SSS SSSS SSSS SSSSSSSSS SSSSSSSSSSSSSSSSSSSS +SSSSSSSSSSSSS SSS SSSS SSSS SSSSSSSSSS SSSSSSSSSSSSSSSSSSSSSS +SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS +SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS +SSSSSSSSSSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS + SSSSSSSSSSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS + SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS + SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS +SSSSSSSSSSSSS SSS SSSSSSSSSSSSSSS SSSS SSSS SSSS SSSS +SSSSSSSSSSSS SSS SSSSSSSSSSSSS SSSS SSSS SSSS SSSS + +""" +_MAINTENANCE_SBATCH_SCRIPT_PATH = dirs.custom_scripts / "perform_maintenance.sh" + +def start_motd(): + """advise in motd that slurm is currently configuring""" + wall_msg = "*** Slurm is currently being configured in the background. ***" + motd_msg = MOTD_HEADER + wall_msg + "\n\n" + Path("/etc/motd").write_text(motd_msg) + util.run(f"wall -n '{wall_msg}'", timeout=30) + + +def end_motd(broadcast=True): + """modify motd to signal that setup is complete""" + Path("/etc/motd").write_text(MOTD_HEADER) + + if not broadcast: + return + + run( + "wall -n '*** Slurm {} setup complete ***'".format(lookup().instance_role), + timeout=30, + ) + if not lookup().is_controller: + run( + """wall -n ' +/home on the controller was mounted over the existing /home. +Log back in to ensure your home directory is correct. +'""", + timeout=30, + ) + + +def failed_motd(): + """modify motd to signal that setup is failed""" + wall_msg = f"*** Slurm setup failed! Please view log: {util.get_log_path()} ***" + motd_msg = MOTD_HEADER + wall_msg + "\n\n" + Path("/etc/motd").write_text(motd_msg) + util.run(f"wall -n '{wall_msg}'", timeout=30) + + +def _startup_script_timeout(lkp: util.Lookup) -> int: + if lkp.is_controller: + return lkp.cfg.get("controller_startup_scripts_timeout", 300) + elif lkp.instance_role == "compute": + return lkp.cfg.get("compute_startup_scripts_timeout", 300) + elif lkp.is_login_node: + return lkp.cfg.login_groups[util.instance_login_group()].get("startup_scripts_timeout", 300) + return 300 + + +def run_custom_scripts(): + """run custom scripts based on instance_role""" + custom_dir = dirs.custom_scripts + if lookup().is_controller: + # controller has all scripts, but only runs controller.d + custom_dirs = [custom_dir / "controller.d"] + elif lookup().instance_role == "compute": + # compute setup with nodeset.d + custom_dirs = [custom_dir / "nodeset.d"] + elif lookup().is_login_node: + # login setup with only login.d + custom_dirs = [custom_dir / "login.d"] + else: + # Unknown role: run nothing + custom_dirs = [] + + timeout = _startup_script_timeout(lookup()) + + custom_scripts = [ + p + for d in custom_dirs + for p in d.rglob("*") + if p.is_file() and not p.name.endswith(".disabled") + ] + print_scripts = ",".join(str(s.relative_to(custom_dir)) for s in custom_scripts) + log.debug(f"custom scripts to run: {custom_dir}/({print_scripts})") + + try: + for script in custom_scripts: + log.info(f"running script {script.name} with timeout={timeout}") + result = run(str(script), timeout=timeout, check=False, shell=True) + runlog = ( + f"{script.name} returncode={result.returncode}\n" + f"stdout={result.stdout}stderr={result.stderr}" + ) + log.info(runlog) + result.check_returncode() + except OSError as e: + log.error(f"script {script} is not executable") + raise e + except subprocess.TimeoutExpired as e: + log.error(f"script {script} did not complete within timeout={timeout}") + raise e + except Exception as e: + log.exception(f"script {script} encountered an exception") + raise e + +def mount_save_state_disk(): + disk_name = f"/dev/disk/by-id/google-{lookup().cfg.controller_state_disk.device_name}" + mount_point = util.slurmdirs.state + fs_type = "ext4" + + rdevice = util.run(f"realpath {disk_name}").stdout.strip() + file_output = util.run(f"file -s {rdevice}").stdout.strip() + if "filesystem" not in file_output: + util.run(f"mkfs -t {fs_type} -q {rdevice}") + + fstab_entry = f"{disk_name} {mount_point} {fs_type}" + with open("/etc/fstab", "r") as f: + fstab = f.readlines() + if fstab_entry not in fstab: + with open("/etc/fstab", "a") as f: + f.write(f"{fstab_entry} defaults 0 0\n") + + util.run(f"systemctl daemon-reload") + + os.makedirs(mount_point, exist_ok=True) + util.run(f"mount {mount_point}") + + util.chown_slurm(mount_point) + + +def setup_jwt_key(): + jwt_key = Path(slurmdirs.state / "jwt_hs256.key") + + if jwt_key.exists(): + log.info("JWT key already exists. Skipping key generation.") + else: + run("dd if=/dev/urandom bs=32 count=1 > " + str(jwt_key), shell=True) + + util.chown_slurm(jwt_key, mode=0o400) + + +def _generate_key(p: Path) -> None: + run(f"dd if=/dev/random of={p} bs=1024 count=1") + + +def setup_key(lkp: util.Lookup) -> None: + file_name = "munge.key" + dir = dirs.munge + + if lkp.cfg.enable_slurm_auth: + file_name = "slurm.key" + dir = slurmdirs.etc + + dst = Path(dir / file_name) + + if lkp.cfg.controller_state_disk.device_name: + # Copy key from persistent state disk + persist = slurmdirs.state / file_name + if not persist.exists(): + _generate_key(persist) + + shutil.copyfile(persist, dst) + if lkp.cfg.enable_slurm_auth: + util.chown_slurm(dst, mode=0o400) + util.chown_slurm(persist, mode=0o400) + else: + shutil.chown(dst, user="munge", group="munge") + os.chmod(dst, stat.S_IRUSR) + else: + if dst.exists(): + log.info("key already exists. Skipping key generation.") + else: + _generate_key(dst) + if lkp.cfg.enable_slurm_auth: + util.chown_slurm(dst, mode=0o400) + else: + shutil.chown(dst, user="munge", group="munge") + os.chmod(dst, stat.S_IRUSR) + + if lkp.cfg.enable_slurm_auth: + # Put key into shared volume for distribution + distributed = util.slurmdirs.key_distribution / file_name + shutil.copyfile(dst, distributed) + util.chown_slurm(distributed, mode=0o400) + # Munge is distributed from /etc/munge. + else: + run("systemctl restart munge", timeout=30) + + +def setup_nss_slurm(): + """install and configure nss_slurm""" + # setup nss_slurm + util.mkdirp(Path("/var/spool/slurmd")) + run( + "ln -s {}/lib/libnss_slurm.so.2 /usr/lib64/libnss_slurm.so.2".format( + slurmdirs.prefix + ), + check=False, + ) + run(r"sed -i 's/\(^\(passwd\|group\):\s\+\)/\1slurm /g' /etc/nsswitch.conf") + + +def setup_sudoers(): + content = """ +# Allow SlurmUser to manage the slurm daemons +slurm ALL= NOPASSWD: /usr/bin/systemctl restart slurmd.service +slurm ALL= NOPASSWD: /usr/bin/systemctl restart sackd.service +slurm ALL= NOPASSWD: /usr/bin/systemctl restart slurmctld.service +""" + sudoers_file = Path("/etc/sudoers.d/slurm") + sudoers_file.write_text(content) + sudoers_file.chmod(0o0440) + + +def setup_maintenance_script(): + perform_maintenance = """#!/bin/bash + +#SBATCH --priority=low +#SBATCH --time=180 + +VM_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google") +ZONE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor: Google" | cut -d '/' -f 4) + +gcloud compute instances perform-maintenance $VM_NAME \ + --zone=$ZONE +""" + + + with open(_MAINTENANCE_SBATCH_SCRIPT_PATH, "w") as f: + f.write(perform_maintenance) + + util.chown_slurm(_MAINTENANCE_SBATCH_SCRIPT_PATH, mode=0o755) + + +def update_system_config(file, content): + """Add system defaults options for service files""" + sysconfig = Path("/etc/sysconfig") + default = Path("/etc/default") + + if sysconfig.exists(): + conf_dir = sysconfig + elif default.exists(): + conf_dir = default + else: + raise Exception("Cannot determine system configuration directory.") + + slurmd_file = Path(conf_dir, file) + slurmd_file.write_text(content) + + +def configure_mysql(): + cnfdir = Path("/etc/my.cnf.d") + if not cnfdir.exists(): + cnfdir = Path("/etc/mysql/conf.d") + if not (cnfdir / "mysql_slurm.cnf").exists(): + (cnfdir / "mysql_slurm.cnf").write_text( + """ +[mysqld] +bind-address=127.0.0.1 +innodb_buffer_pool_size=1024M +innodb_log_file_size=64M +innodb_lock_wait_timeout=900 +""" + ) + run("systemctl enable mariadb", timeout=30) + run("systemctl restart mariadb", timeout=30) + + mysql, host = "mysql -u root -e", lookup().control_host + run(f"""{mysql} "drop user if exists 'slurm'@'localhost'";""", timeout=30) + run(f"""{mysql} "create user 'slurm'@'localhost'";""", timeout=30) + run(f"""{mysql} "grant all on slurm_acct_db.* TO 'slurm'@'localhost'";""", timeout=30) + run(f"""{mysql} "drop user if exists 'slurm'@'{host}'";""", timeout=30) + run(f"""{mysql} "create user 'slurm'@'{host}'";""", timeout=30) + run(f"""{mysql} "grant all on slurm_acct_db.* TO 'slurm'@'{host}'";""", timeout=30) + + +def configure_dirs(): + for p in dirs.values(): + util.mkdirp(p) + + for p in (dirs.slurm, dirs.scripts, dirs.custom_scripts): + util.chown_slurm(p) + + for p in slurmdirs.values(): + util.mkdirp(p) + util.chown_slurm(p) + + for sl, tgt in ( # create symlinks + (Path("/etc/slurm"), slurmdirs.etc), + (dirs.scripts / "etc", slurmdirs.etc), + (dirs.scripts / "log", dirs.log), + ): + if sl.exists() and sl.is_symlink(): + sl.unlink() + sl.symlink_to(tgt) + + # copy auxiliary scripts + for dst_folder, src_file in ((lookup().cfg.slurm_bin_dir, + Path("sort_nodes.py")), + (dirs.custom_scripts / "task_prolog.d", + Path("tools/task-prolog")), + (dirs.custom_scripts / "task_epilog.d", + Path("tools/task-epilog"))): + dst = Path(dst_folder) / src_file.name + util.mkdirp(dst.parent) + shutil.copyfile(util.scripts_dir / src_file, dst) + os.chmod(dst, 0o755) + + +def self_report_controller_address(lkp: util.Lookup) -> None: + if not lkp.cfg.controller_network_attachment: + return # only self report address if network attachment is used + data = { "slurm_control_addr": lkp.cfg.slurm_control_addr } + bucket, prefix = util._get_bucket_and_common_prefix() + blob = util.storage_client().bucket(bucket).blob(f"{prefix}/controller_addr.yaml") + with blob.open('w') as f: + f.write(yaml.dump(data)) + +def setup_controller(): + """Run controller setup""" + log.info("Setting up controller") + lkp = util.lookup() + util.chown_slurm(dirs.scripts / "config.yaml", mode=0o600) + install_custom_scripts() + conf.gen_controller_configs(lkp) + + if lkp.cfg.controller_state_disk.device_name != None: + mount_save_state_disk() + + setup_jwt_key() + setup_key(lkp) + + setup_sudoers() + setup_network_storage() + + run_custom_scripts() + + if not lkp.cfg.cloudsql_secret: + configure_mysql() + + run("systemctl enable slurmdbd", timeout=30) + run("systemctl restart slurmdbd", timeout=30) + + # Wait for slurmdbd to come up + time.sleep(5) + + sacctmgr = f"{slurmdirs.prefix}/bin/sacctmgr -i" + result = run( + f"{sacctmgr} add cluster {lkp.cfg.slurm_cluster_name}", timeout=30, check=False + ) + if "already exists" in result.stdout: + log.info(result.stdout) + elif result.returncode > 1: + result.check_returncode() # will raise error + + run("systemctl enable slurmctld", timeout=30) + run("systemctl restart slurmctld", timeout=30) + + run("systemctl enable slurmrestd", timeout=30) + run("systemctl restart slurmrestd", timeout=30) + + # Export at the end to signal that everything is up + run("systemctl enable nfs-server", timeout=30) + run("systemctl start nfs-server", timeout=30) + + setup_nfs_exports() + run("systemctl enable --now slurmcmd.timer", timeout=30) + + log.info("Check status of cluster services") + if not lkp.cfg.enable_slurm_auth: + run("systemctl status munge", timeout=30) + run("systemctl status slurmdbd", timeout=30) + run("systemctl status slurmctld", timeout=30) + run("systemctl status slurmrestd", timeout=30) + + try: + slurmsync.sync_instances() + except Exception: + log.exception("Failed to sync instances, will try next time.") + + run("systemctl enable slurm_load_bq.timer", timeout=30) + run("systemctl start slurm_load_bq.timer", timeout=30) + run("systemctl status slurm_load_bq.timer", timeout=30) + + # Add script to perform maintenance + setup_maintenance_script() + + self_report_controller_address(lkp) + + log.info("Done setting up controller") + pass + + +def setup_login(): + """run login node setup""" + log.info("Setting up login") + + lkp = lookup() + slurmctld_host = f"{lkp.control_host}" + if lkp.control_addr: + slurmctld_host = f"{lkp.control_host}({lkp.control_addr})" + sackd_options = [ + f'--conf-server="{slurmctld_host}:{lkp.control_host_port}"', + ] + sysconf = f"""SACKD_OPTIONS='{" ".join(sackd_options)}'""" + update_system_config("sackd", sysconf) + install_custom_scripts() + + setup_network_storage() + setup_sudoers() + if not lkp.cfg.enable_slurm_auth: + run("systemctl restart munge", timeout=30) + run("systemctl enable sackd", timeout=30) + run("systemctl restart sackd", timeout=30) + run("systemctl enable --now slurmcmd.timer", timeout=30) + + run_custom_scripts() + + log.info("Check status of cluster services") + if not lkp.cfg.enable_slurm_auth: + run("systemctl status munge", timeout=30) + run("systemctl status sackd", timeout=30) + + log.info("Done setting up login") + + +def setup_compute(): + """run compute node setup""" + log.info("Setting up compute") + + lkp = lookup() + util.chown_slurm(dirs.scripts / "config.yaml", mode=0o600) + slurmctld_host = f"{lkp.control_host}" + if lkp.control_addr: + slurmctld_host = f"{lkp.control_host}({lkp.control_addr})" + slurmd_options = [ + f'--conf-server="{slurmctld_host}:{lkp.control_host_port}"', + ] + + try: + slurmd_feature = util.instance_metadata("attributes/slurmd_feature", silent=True) + except util.MetadataNotFoundError: + slurmd_feature = None + + if slurmd_feature is not None: + slurmd_options.append(f'--conf="Feature={slurmd_feature}"') + slurmd_options.append("-Z") + + sysconf = f"""SLURMD_OPTIONS='{" ".join(slurmd_options)}'""" + update_system_config("slurmd", sysconf) + install_custom_scripts() + + setup_nss_slurm() + setup_network_storage() + + has_gpu = run("lspci | grep --ignore-case 'NVIDIA' | wc -l", shell=True).returncode + if has_gpu: + run("nvidia-smi") + + run_custom_scripts() + + setup_sudoers() + if not lkp.cfg.enable_slurm_auth: + run("systemctl restart munge", timeout=30) + run("systemctl enable slurmd", timeout=30) + run("systemctl restart slurmd", timeout=30) + run("systemctl enable --now slurmcmd.timer", timeout=30) + + log.info("Check status of cluster services") + if not lkp.cfg.enable_slurm_auth: + run("systemctl status munge", timeout=30) + run("systemctl status slurmd", timeout=30) + + log.info("Done setting up compute") + +def setup_cloud_ops() -> None: + """Add health checks, deployment info, and updated setup path to cloud ops config.""" + cloudOpsStatus = run( + "systemctl is-active --quiet google-cloud-ops-agent.service", check=False + ).returncode + + if cloudOpsStatus != 0: + return + + with open("/etc/google-cloud-ops-agent/config.yaml", "r") as f: + file = yaml.safe_load(f) + + # Update setup receiver path + file["logging"]["receivers"]["setup"]["include_paths"] = ["/var/log/slurm/setup.log"] + + # Add chs_health_check receiver + file["logging"]["receivers"]["chs_health_check"] = { + "type": "files", + "include_paths": ["/var/log/slurm/chs_health_check.log"], + "record_log_file_path": True, + } + + cluster_info = { + 'type':'modify_fields', + 'fields': { + 'labels."cluster_name"':{ + 'static_value':f"{lookup().cfg.slurm_cluster_name}" + }, + 'labels."hostname"':{ + 'static_value': f"{lookup().hostname}" + } + } + } + + file["logging"]["processors"]["add_cluster_info"] = cluster_info + file["logging"]["service"]["pipelines"]["slurmlog_pipeline"]["processors"].append("add_cluster_info") + file["logging"]["service"]["pipelines"]["slurmlog2_pipeline"]["processors"].append("add_cluster_info") + + # Add chs_health_check to slurmlog2_pipeline + file["logging"]["service"]["pipelines"]["slurmlog2_pipeline"]["receivers"].append( + "chs_health_check" + ) + + with open("/etc/google-cloud-ops-agent/config.yaml", "w") as f: + yaml.safe_dump(file, f, sort_keys=False) + + try: + run("systemctl restart google-cloud-ops-agent.service", timeout=30) + except subprocess.TimeoutExpired: + log.error("google-cloud-ops-agent.service did not restart within 30s.") + result=run("journalctl -u google-cloud-ops-agent-opentelemetry-collector.service", timeout=30, shell=True) #Gives more in-depth logs than cloud ops agent + if result.stdout: + log.error(f"System logs for google-cloud-ops-agent-opentelemetry-collector.service:\n{result.stdout}") + raise + + +def main(): + start_motd() + + log.info("Starting setup, fetching config") + sleep_seconds = 5 + while True: + try: + _, cfg = util.fetch_config() + util.update_config(cfg) + break + except util.DeffetiveStoredConfigError as e: + log.warning(f"config is not ready yet: {e}, sleeping for {sleep_seconds}s") + except Exception as e: + log.exception(f"unexpected error while fetching config, sleeping for {sleep_seconds}s") + time.sleep(sleep_seconds) + log.info("Config fetched") + setup_cloud_ops() + configure_dirs() + # call the setup function for the instance type + { + "controller": setup_controller, + "compute": setup_compute, + "login": setup_login, + }.get( + lookup().instance_role, + lambda: log.fatal(f"Unknown node role: {lookup().instance_role}"))() + + end_motd() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--slurmd-feature", dest="slurmd_feature", help="Unused, to be removed.") + _ = util.init_log_and_parse(parser) + + try: + main() + except Exception: + log.exception("Aborting setup...") + failed_motd() diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py new file mode 100644 index 0000000000..e2561dffa1 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py @@ -0,0 +1,327 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +import os +import sys +import stat +import time +import logging +import uuid + +import shutil +from pathlib import Path +from concurrent.futures import as_completed +from addict import Dict as NSDict # type: ignore + +import util +from util import NSMount, lookup, run, dirs, separate +from more_executors import Executors, ExceptionRetryPolicy + + +log = logging.getLogger() + +def mounts_by_local(mounts: list[NSMount]) -> dict[str, NSMount]: + """convert list of mounts to dict of mounts, local_mount as key""" + return {str(m.local_mount.resolve()): m for m in mounts} + + +def _get_default_mounts(lkp: util.Lookup) -> list[NSMount]: + if lkp.cfg.disable_default_mounts: + return [] + return [ + NSMount( + server_ip=lkp.controller_mount_server_ip(), + remote_mount=path, + local_mount=path, + fs_type="nfs", + mount_options="defaults,hard,intr", + ) + for path in ( + dirs.home, + dirs.apps, + ) + ] + +def get_slurm_bucket_mount() -> NSMount: + bucket, path = util._get_bucket_and_common_prefix() + return NSMount( + fs_type="gcsfuse", + server_ip="", + remote_mount=Path(bucket), + local_mount=dirs.slurm_bucket_mount, + mount_options=f"defaults,_netdev,implicit_dirs,only_dir={path}", + ) + +def resolve_network_storage() -> List[NSMount]: + """Combine appropriate network_storage fields to a single list""" + lkp = lookup() + + # create dict of mounts, local_mount: mount_info + mounts = mounts_by_local(_get_default_mounts(lkp)) + + if lkp.is_controller and util.should_mount_slurm_bucket(): + mounts.update(mounts_by_local([get_slurm_bucket_mount()])) + + # On non-controller instances, entries in network_storage could overwrite + # default exports from the controller. Be careful, of course + common = [lkp.normalize_ns_mount(m) for m in lkp.cfg.network_storage] + mounts.update(mounts_by_local(common)) + + if lkp.is_login_node: + login_group = lkp.cfg.login_groups[util.instance_login_group()] + login_ns = [lkp.normalize_ns_mount(m) for m in login_group.network_storage] + mounts.update(mounts_by_local(login_ns)) + + if lkp.instance_role == "compute": + try: + nodeset = lkp.node_nodeset() + except Exception: + pass # external nodename, skip lookup + else: + nodeset_ns = [lkp.normalize_ns_mount(m) for m in nodeset.network_storage] + mounts.update(mounts_by_local(nodeset_ns)) + + return list(mounts.values()) + + +def is_controller_mount(mount) -> bool: + # NOTE: Valid Lustre server_ip can take the form of '@tcp' + server_ip = mount.server_ip.split("@")[0] + mount_addr = util.host_lookup(server_ip) + return mount_addr == lookup().control_host_addr + +def setup_network_storage(): + """prepare network fs mounts and add them to fstab""" + log.info("Set up network storage") + + all_mounts = resolve_network_storage() + if lookup().is_controller: + mounts, _ = separate(is_controller_mount, all_mounts) + else: + mounts = all_mounts + + # Determine fstab entries and write them out + fstab_entries = [] + for mount in mounts: + local_mount = mount.local_mount + fs_type = mount.fs_type + server_ip = mount.server_ip or "" + src = mount.remote_mount if fs_type == "gcsfuse" else f"{server_ip}:{mount.remote_mount}" + + log.info(f"Setting up mount ({fs_type}) {src} to {local_mount}") + util.mkdirp(local_mount) + + mount_options = mount.mount_options.split(",") if mount.mount_options else [] + if "_netdev" not in mount_options: + mount_options += ["_netdev"] + options_line = ",".join(mount_options) + + + fstab_entries.append(f"{src} {local_mount} {fs_type} {options_line} 0 0") + + fstab = Path("/etc/fstab") + if not Path(fstab.with_suffix(".bak")).is_file(): + shutil.copy2(fstab, fstab.with_suffix(".bak")) + shutil.copy2(fstab.with_suffix(".bak"), fstab) + with open(fstab, "a") as f: + f.write("\n") + for entry in fstab_entries: + f.write(entry) + f.write("\n") + + mount_fstab(mounts, log) + if lookup().cfg.enable_slurm_auth: + slurm_key_mount_handler() + else: + munge_mount_handler() + + +def mount_fstab(mounts: list[NSMount], log): + """Wait on each mount, then make sure all fstab is mounted""" + def mount_path(path: Path): + log.info(f"Waiting for '{path}' to be mounted...") + try: + run(f"mount {path}", timeout=120) + except Exception as e: + exc_type, _, _ = sys.exc_info() + log.error(f"mount of path '{path}' failed: {exc_type}: {e}") + raise e + log.info(f"Mount point '{path}' was mounted.") + + MAX_MOUNT_TIMEOUT = 60 * 5 + future_list = [] + retry_policy = ExceptionRetryPolicy( + max_attempts=40, exponent=1.6, sleep=1.0, max_sleep=16.0 + ) + with Executors.thread_pool().with_timeout(MAX_MOUNT_TIMEOUT).with_retry( + retry_policy=retry_policy + ) as exe: + for m in mounts: + future = exe.submit(mount_path, m.local_mount) + future_list.append(future) + + # Iterate over futures, checking for exceptions + for future in as_completed(future_list): + try: + future.result() + except Exception as e: + raise e + + +def munge_mount_handler(): + if lookup().is_controller: + return + mnt = lookup().munge_mount + + log.info(f"Mounting munge share to: {mnt.local_mount}") + mnt.local_mount.mkdir() + if mnt.fs_type == "gcsfuse": + cmd = [ + "gcsfuse", + f"--only-dir={mnt.remote_mount}" if mnt.remote_mount != "" else None, + mnt.server_ip, + str(mnt.local_mount), + ] + else: + cmd = [ + "mount", + f"--types={mnt.fs_type}", + f"--options={mnt.mount_options}" if mnt.mount_options != "" else None, + f"{mnt.server_ip}:{mnt.remote_mount}", + str(mnt.local_mount), + ] + # wait max 120s for munge mount + timeout = 120 + for retry, wait in enumerate(util.backoff_delay(0.5, timeout), 1): + try: + run(cmd, timeout=timeout) + break + except Exception as e: + log.error( + f"munge mount failed: '{cmd}' {e}, try {retry}, waiting {wait:0.2f}s" + ) + time.sleep(wait) + err = e + continue + else: + raise err + + munge_key = Path(dirs.munge / "munge.key") + log.info(f"Copy munge.key from: {mnt.local_mount}") + shutil.copy2(Path(mnt.local_mount / "munge.key"), munge_key) + + log.info("Restrict permissions of munge.key") + shutil.chown(munge_key, user="munge", group="munge") + os.chmod(munge_key, stat.S_IRUSR) + + log.info(f"Unmount {mnt.local_mount}") + if mnt.fs_type == "gcsfuse": + run(f"fusermount -u {mnt.local_mount}", timeout=120) + else: + run(f"umount {mnt.local_mount}", timeout=120) + shutil.rmtree(mnt.local_mount) + +def slurm_key_mount_handler(): + if lookup().is_controller: + return + mnt = lookup().slurm_key_mount + + log.info(f"Mounting slurm_key share to: {mnt.local_mount}") + if mnt.fs_type == "gcsfuse": + cmd = [ + "gcsfuse", + f"--only-dir={mnt.remote_mount}" if mnt.remote_mount != "" else None, + mnt.server_ip, + str(mnt.local_mount), + ] + else: + cmd = [ + "mount", + f"--types={mnt.fs_type}", + f"--options={mnt.mount_options}" if mnt.mount_options != "" else None, + f"{mnt.server_ip}:{mnt.remote_mount}", + str(mnt.local_mount), + ] + timeout = 120 # wait max 120s to mount + for retry, wait in enumerate(util.backoff_delay(0.5, timeout), 1): + try: + run(cmd, timeout=timeout) + break + except Exception as e: + log.error( + f"slurm key mount failed: '{cmd}' {e}, try {retry}, waiting {wait:0.2f}s" + ) + time.sleep(wait) + err = e + continue + else: + raise err + + file_name = "slurm.key" + dst = Path(util.slurmdirs.etc / file_name) + log.info(f"Copy slurm.key from: {mnt.local_mount}") + shutil.copy2(mnt.local_mount / file_name, dst) + + log.info("Restrict permissions of slurm.key") + util.chown_slurm(dst, mode=0o400) + + log.info(f"Unmount {mnt.local_mount}") + if mnt.fs_type == "gcsfuse": + run(f"fusermount -u {mnt.local_mount}", timeout=120) + else: + run(f"umount {mnt.local_mount}", timeout=120) + shutil.rmtree(mnt.local_mount) + + +def setup_nfs_exports(): + """nfs export all needed directories""" + lkp = util.lookup() + assert lkp.is_controller + + # The controller only needs to set up exports for cluster-internal mounts + exported_mounts = [m for m in resolve_network_storage() if is_controller_mount(m)] + + # key by remote mount path since that is what needs exporting + to_export = {m.remote_mount: "*(rw,no_subtree_check,no_root_squash)" for m in exported_mounts} + + key_mount = lkp.slurm_key_mount if lkp.cfg.enable_slurm_auth else lkp.munge_mount + if is_controller_mount(key_mount): + # Export key mount as read-only + to_export[key_mount.remote_mount] = "*(ro,no_subtree_check,no_root_squash)" + + if util.should_mount_slurm_bucket(): + mnt = get_slurm_bucket_mount() + # FSID is required for virtual filesystem that is not based on a device + # Also export it as read-only + fsid=str(uuid.uuid4()) + to_export[mnt.local_mount] = f"*(ro,no_subtree_check,no_root_squash,fsid={fsid})" + + # export path if corresponding selector boolean is True + lines = [] + for path,options in to_export.items(): + util.mkdirp(Path(path)) + run(rf"sed -i '\#{path}#d' /etc/exports", timeout=30) + lines.append(f"{path} {options}") + + exportsd = Path("/etc/exports.d") + util.mkdirp(exportsd) + with (exportsd / "slurm.exports").open("w") as f: + f.write("\n") + f.write("\n".join(lines)) + run("exportfs -a", timeout=30) diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py new file mode 100644 index 0000000000..7285096c7c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py @@ -0,0 +1,661 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import fcntl +import json +import logging +import re +import sys +import shlex +from datetime import datetime, timedelta +from itertools import chain +from pathlib import Path +from dataclasses import dataclass +from typing import Dict, Tuple, List, Optional, Protocol, Any +from functools import lru_cache + +import util +from util import ( + batch_execute, + ensure_execute, + execute_with_futures, + FutureReservation, + install_custom_scripts, + run, + separate, + to_hostlist, + NodeState, + chunked, + dirs, +) +from util import lookup +from suspend import delete_instances +import tpu +import conf +import watch_delete_vm_op + +log = logging.getLogger() + +TOT_REQ_CNT = 1000 +_MAINTENANCE_SBATCH_SCRIPT_PATH = dirs.custom_scripts / "perform_maintenance.sh" + +class NodeAction(Protocol): + def apply(self, nodes:List[str]) -> None: + ... + + def __hash__(self): + ... + +@dataclass(frozen=True) +class NodeActionPowerUp(): + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} instances to resume ({hostlist})") + run(f"{lookup().scontrol} update nodename={hostlist} state=power_up") + +@dataclass(frozen=True) +class NodeActionIdle(): + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} nodes to idle ({hostlist})") + run(f"{lookup().scontrol} update nodename={hostlist} state=resume") + +@dataclass(frozen=True) +class NodeActionPowerDown(): + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} instances to power down ({hostlist})") + run(f"{lookup().scontrol} update nodename={hostlist} state=power_down") + +@dataclass(frozen=True) +class NodeActionDelete(): + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} instances to delete ({hostlist})") + delete_instances(nodes) + +@dataclass(frozen=True) +class NodeActionPrempt(): + def apply(self, nodes:List[str]) -> None: + NodeActionDown(reason="Preempted instance").apply(nodes) + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} instances restarted ({hostlist})") + start_instances(nodes) + +@dataclass(frozen=True) +class NodeActionUnchanged(): + def apply(self, nodes:List[str]) -> None: + pass + +@dataclass(frozen=True) +class NodeActionDown(): + reason: str + + def apply(self, nodes: List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.info(f"{len(nodes)} nodes set down ({hostlist}) with reason={self.reason}") + run(f"{lookup().scontrol} update nodename={hostlist} state=down reason={shlex.quote(self.reason)}") + +@dataclass(frozen=True) +class NodeActionUnknown(): + slurm_state: Optional[NodeState] + instance_state: Optional[str] + + def apply(self, nodes:List[str]) -> None: + hostlist = util.to_hostlist(nodes) + log.error(f"{len(nodes)} nodes have unexpected {self.slurm_state} and instance state:{self.instance_state}, ({hostlist})") + +def start_instance_op(node: str) -> Any: + inst = lookup().instance(node) + assert inst + + return lookup().compute.instances().start( + project=lookup().project, + zone=inst.zone, + instance=inst, + ) + + +def start_instances(node_list): + log.info("{} instances to start ({})".format(len(node_list), ",".join(node_list))) + lkp = lookup() + # TODO: use code from resume.py to assign proper placement + normal, tpu_nodes = separate(lkp.node_is_tpu, node_list) + ops = {node: start_instance_op(node) for node in normal} + + done, failed = batch_execute(ops) + + tpu_start_data = [] + for ns, nodes in util.groupby_unsorted(tpu_nodes, lkp.node_nodeset_name): + tpuobj = tpu.TPU.make(ns, lkp) + for snodes in chunked(nodes, n=tpuobj.vmcount): + tpu_start_data.append({"tpu": tpuobj, "node": snodes}) + execute_with_futures(tpu.start_tpu, tpu_start_data) + + +def _find_dynamic_node_status() -> NodeAction: + # TODO: cover more cases: + # * delete dead dynamic nodes + # * delete orhpaned instances + return NodeActionUnchanged() # don't touch dynamic nodes + +def get_fr_action(fr: FutureReservation, state:Optional[NodeState]) -> Optional[NodeAction]: + now = util.now() + if state is None: + return None # handle like any other node + if fr.start_time < now < fr.end_time: + return None # handle like any other node + + if state.base == "DOWN": + return NodeActionUnchanged() + if fr.start_time >= now: + msg = f"Waiting for reservation:{fr.name} to start at {fr.start_time}" + else: + msg = f"Reservation:{fr.name} is after its end-time" + return NodeActionDown(reason=msg) + +def _find_tpu_node_action(nodename, state) -> NodeAction: + lkp = lookup() + tpuobj = tpu.TPU.make(lkp.node_nodeset_name(nodename), lkp) + inst = tpuobj.get_node(nodename) + # If we do not find the node but it is from a Tpu that has multiple vms look for the master node + if inst is None and tpuobj.vmcount > 1: + # Get the tpu slurm nodelist of the nodes in the same tpu group as nodename + nodelist = run( + f"{lkp.scontrol} show topo {nodename}" + + " | awk -F'=' '/Level=0/ { print $NF }'", + shell=True, + ).stdout + l_nodelist = util.to_hostnames(nodelist) + group_names = set(l_nodelist) + # get the list of all the existing tpus in the nodeset + tpus_list = set(tpuobj.list_node_names()) + # In the intersection there must be only one node that is the master + tpus_int = list(group_names.intersection(tpus_list)) + if len(tpus_int) > 1: + log.error( + f"More than one cloud tpu node for tpu group {nodelist}, there should be only one that should be {l_nodelist[0]}, but we have found {tpus_int}" + ) + return NodeActionUnknown(slurm_state=state, instance_state=None) + if len(tpus_int) == 1: + inst = tpuobj.get_node(tpus_int[0]) + # if len(tpus_int ==0) this case is not relevant as this would be the case always that a TPU group is not running + if inst is None: + if state.base == "DOWN" and "POWERED_DOWN" in state.flags: + return NodeActionIdle() + if "POWERING_DOWN" in state.flags: + return NodeActionIdle() + if "COMPLETING" in state.flags: + return NodeActionDown(reason="Unbacked instance") + if state.base != "DOWN" and not ( + set(("POWER_DOWN", "POWERING_UP", "POWERING_DOWN", "POWERED_DOWN")) + & state.flags + ): + return NodeActionDown(reason="Unbacked instance") + if lkp.is_static_node(nodename): + return NodeActionPowerUp() + elif ( + state is not None + and "POWERED_DOWN" not in state.flags + and "POWERING_DOWN" not in state.flags + and inst.state == tpu.TPU.State.STOPPED + ): + if tpuobj.preemptible: + return NodeActionPrempt() + if state.base != "DOWN": + return NodeActionDown(reason="Instance terminated") + elif ( + state is None or "POWERED_DOWN" in state.flags + ) and inst.state == tpu.TPU.State.READY: + return NodeActionDelete() + elif state is None: + # if state is None here, the instance exists but it's not in Slurm + return NodeActionUnknown(slurm_state=state, instance_state=inst.status) + + return NodeActionUnchanged() + +def get_node_action(nodename: str) -> NodeAction: + """Determine node/instance status that requires action""" + lkp = lookup() + state = lkp.node_state(nodename) + + if lkp.node_is_fr(nodename): + fr = lkp.future_reservation(lkp.node_nodeset(nodename)) + assert fr + if action := get_fr_action(fr, state): + return action + + if lkp.node_is_dyn(nodename): + return _find_dynamic_node_status() + + if lkp.node_is_tpu(nodename): + return _find_tpu_node_action(nodename, state) + + # split below is workaround for VMs whose hostname is FQDN + inst = lkp.instance(nodename.split(".")[0]) + power_flags = frozenset( + ("POWER_DOWN", "POWERING_UP", "POWERING_DOWN", "POWERED_DOWN") + ) & (state.flags if state is not None else set()) + + if (state is None) and (inst is None): + # Should never happen + return NodeActionUnknown(None, None) + if inst is None: + assert state is not None # to keep type-checker happy + if "POWERING_UP" in state.flags: + return NodeActionUnchanged() + if state.base == "DOWN" and "POWERED_DOWN" in state.flags: + return NodeActionIdle() + if "POWERING_DOWN" in state.flags: + return NodeActionIdle() + if "COMPLETING" in state.flags: + return NodeActionDown(reason="Unbacked instance") + if state.base != "DOWN" and not power_flags: + return NodeActionDown(reason="Unbacked instance") + if state.base == "DOWN" and not power_flags: + return NodeActionPowerDown() + if "POWERED_DOWN" in state.flags and lkp.is_static_node(nodename): + return NodeActionPowerUp() + elif ( + state is not None + and "POWERED_DOWN" not in state.flags + and "POWERING_DOWN" not in state.flags + and inst.status == "TERMINATED" + ): + if inst.scheduling.preemptible: + return NodeActionPrempt() + if state.base != "DOWN": + return NodeActionDown(reason="Instance terminated") + elif (state is None or "POWERED_DOWN" in state.flags) and inst.status == "RUNNING": + log.info("%s is potential orphan node", nodename) + threshold = timedelta(seconds=90) + age = util.now() - inst.creation_timestamp + log.info(f"{nodename} state: {state}, age: {age}") + if age < threshold: + log.info(f"{nodename} not marked as orphan, it started less than {threshold.seconds}s ago ({age.seconds}s)") + return NodeActionUnchanged() + return NodeActionDelete() + elif state is None: + # if state is None here, the instance exists but it's not in Slurm + return NodeActionUnknown(slurm_state=state, instance_state=inst.status) + + return NodeActionUnchanged() + + +def delete_resource_policies(links: list[str], lkp: util.Lookup) -> None: + requests = {} + for link in links: + name = util.trim_self_link(link) + region = util.parse_self_link(link).region + requests[name] = lkp.compute.resourcePolicies().delete(project=lkp.project, region=region, resourcePolicy=name) + + def swallow_err(_: str) -> None: + pass + + done, failed = batch_execute(requests, log_err=swallow_err) + if failed: + # Filter out resourceInUseByAnotherResource errors , they are expected to happen + def ignore_err(e) -> bool: + return "resourceInUseByAnotherResource" in str(e) + + failures = [f"{n}: {e}" for n, (_, e) in failed.items() if not ignore_err(e)] + if failures: + log.error(f"some placement groups failed to delete: {failures}") + log.info( + f"deleted {len(done)} of {len(links)} placement groups ({to_hostlist(done.keys())})" + ) + + + +@lru_cache +def _get_resource_policies_in_region(lkp: util.Lookup, region: str) -> list[Any]: + res = [] + act = lkp.compute.resourcePolicies() + op = act.list(project=lkp.project, region=region) + prefix = f"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-" + while op is not None: + result = ensure_execute(op) + res.extend([p for p in result.get("items", []) if p.get("name", "").startswith(prefix)]) + op = act.list_next(op, result) + return res + + +@lru_cache +def _get_resource_policies(lkp: util.Lookup) -> list[Any]: + res = [] + for region in lkp.cluster_regions(): + res.extend(_get_resource_policies_in_region(lkp, region)) + return res + +def sync_placement_groups(): + """Delete placement policies that are for jobs that have completed/terminated""" + keep_states = frozenset( + [ + "RUNNING", + "CONFIGURING", + "STOPPED", + "SUSPENDED", + "COMPLETING", + "PENDING", + ] + ) + + lkp = lookup() + keep_jobs = { + str(job.id) + for job in lkp.get_jobs() + if job.job_state in keep_states + } + keep_jobs.add("0") # Job 0 is a placeholder for static node placement + + to_delete = [] + pg_regex = re.compile( + rf"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-(?P[^\s\-]+)-(?P\d+)-(?P\d+)" + ) + + for pg in _get_resource_policies(lkp): + name = pg["name"] + + if (mtch := pg_regex.match(name)) is None: + log.warning(f"Unexpected resource policy {name=}") + continue + if mtch.group("job_id") not in keep_jobs: + to_delete.append(pg["selfLink"]) + + if to_delete: + delete_resource_policies(to_delete, lkp) + + +def sync_instances(): + compute_instances = { + name for name, inst in lookup().instances().items() if inst.role == "compute" + } + slurm_nodes = set(lookup().slurm_nodes().keys()) + log.debug(f"reconciling {len(compute_instances)} GCP instances and {len(slurm_nodes)} Slurm nodes.") + + for action, nodes in util.groupby_unsorted(list(compute_instances | slurm_nodes), get_node_action): + action.apply(list(nodes)) + + +def reconfigure_slurm(): + update_msg = "*** slurm configuration was updated ***" + if lookup().cfg.hybrid: + # terraform handles generating the config.yaml, don't do it here + return + + upd, cfg_new = util.fetch_config() + if not upd: + log.debug("No changes in config detected.") + return + log.debug("Changes in config detected. Reconfiguring Slurm now.") + util.update_config(cfg_new) + + if lookup().is_controller: + conf.gen_controller_configs(lookup()) + log.info("Restarting slurmctld to make changes take effect.") + try: + # TODO: consider removing "restart" since "reconfigure" should restart slurmctld as well + run("sudo systemctl restart slurmctld.service", check=False) + util.scontrol_reconfigure(lookup()) + except Exception: + log.exception("failed to reconfigure slurmctld") + util.run(f"wall '{update_msg}'", timeout=30) + log.debug("Done.") + elif lookup().instance_role_safe == "compute": + log.info("Restarting slurmd to make changes take effect.") + run("systemctl restart slurmd") + util.run(f"wall '{update_msg}'", timeout=30) + log.debug("Done.") + elif lookup().is_login_node: + log.info("Restarting sackd to make changes take effect.") + run("systemctl restart sackd") + util.run(f"wall '{update_msg}'", timeout=30) + log.debug("Done.") + + +def update_topology(lkp: util.Lookup) -> None: + if conf.topology_plugin(lkp) != conf.TOPOLOGY_PLUGIN_TREE: + return + updated, summary = conf.gen_topology_conf(lkp) + if updated: + log.info("Topology configuration updated. Reconfiguring Slurm.") + util.scontrol_reconfigure(lkp) + # Safe summary only after Slurm got reconfigured, so summary reflects Slurm POV + summary.dump(lkp) + + +def delete_reservation(lkp: util.Lookup, reservation_name: str) -> None: + util.run(f"{lkp.scontrol} delete reservation {reservation_name}") + + +def create_reservation(lkp: util.Lookup, reservation_name: str, node: str, start_time: datetime) -> None: + # Format time to be compatible with slurm reservation. + formatted_start_time = start_time.strftime('%Y-%m-%dT%H:%M:%S') + + util.run(f"{lkp.scontrol} create reservation user=slurm starttime={formatted_start_time} duration=180 nodes={node} reservationname={reservation_name} flags=maint,ignore_jobs") + + +def get_slurm_reservation_maintenance(lkp: util.Lookup) -> Dict[str, datetime]: + res = util.run(f"{lkp.scontrol} show reservation --json") + all_reservations = json.loads(res.stdout) + reservation_map = {} + + for reservation in all_reservations['reservations']: + name = reservation.get('name') + nodes = reservation.get('node_list') + time_epoch = reservation.get('start_time', {}).get('number') + + if name is None or nodes is None or time_epoch is None: + continue + + if reservation.get('node_count') != 1: + continue + + if name != f"{nodes}_maintenance": + continue + + reservation_map[name] = datetime.fromtimestamp(time_epoch) + + return reservation_map + +@lru_cache +def get_upcoming_maintenance(lkp: util.Lookup) -> Dict[str, Tuple[str, datetime]]: + upc_maint_map = {} + + for node, inst in lkp.instances().items(): + if inst.resource_status.upcoming_maintenance: + upc_maint_map[node + "_maintenance"] = (node, inst.resource_status.upcoming_maintenance.window_start_time) + + return upc_maint_map + + +def sync_maintenance_reservation(lkp: util.Lookup) -> None: + upc_maint_map = get_upcoming_maintenance(lkp) # map reservation_name -> (node_name, time) + log.debug(f"upcoming-maintenance-vms: {upc_maint_map}") + + curr_reservation_map = get_slurm_reservation_maintenance(lkp) # map reservation_name -> time + log.debug(f"curr-reservation-map: {curr_reservation_map}") + + del_reservation = set(curr_reservation_map.keys() - upc_maint_map.keys()) + create_reservation_map = {} + + for res_name, (node, start_time) in upc_maint_map.items(): + try: + enabled = lkp.node_nodeset(node).enable_maintenance_reservation + except Exception: + enabled = False + + if not enabled: + if res_name in curr_reservation_map: + del_reservation.add(res_name) + continue + + if res_name in curr_reservation_map: + diff = curr_reservation_map[res_name] - start_time + if abs(diff) <= timedelta(seconds=1): + continue + else: + del_reservation.add(res_name) + create_reservation_map[res_name] = (node, start_time) + else: + create_reservation_map[res_name] = (node, start_time) + + log.debug(f"del-reservation: {del_reservation}") + for res_name in del_reservation: + delete_reservation(lkp, res_name) + + log.debug(f"create-reservation-map: {create_reservation_map}") + for res_name, (node, start_time) in create_reservation_map.items(): + create_reservation(lkp, res_name, node, start_time) + + +def delete_maintenance_job(job_name: str) -> None: + util.run(f"scancel --name={job_name}") + + +def create_maintenance_job(job_name: str, node: str) -> None: + util.run(f"sbatch --job-name={job_name} --nodelist={node} {_MAINTENANCE_SBATCH_SCRIPT_PATH}") + + +def get_slurm_maintenance_job(lkp: util.Lookup) -> Dict[str, str]: + jobs = {} + + for job in lkp.get_jobs(): + if job.name is None or job.required_nodes is None or job.job_state is None: + continue + + if job.name != f"{job.required_nodes}_maintenance": + continue + + if job.job_state != "PENDING": + continue + + jobs[job.name] = job.required_nodes + + return jobs + + +def sync_opportunistic_maintenance(lkp: util.Lookup) -> None: + upc_maint_map = get_upcoming_maintenance(lkp) # map job_name -> (node_name, time) + log.debug(f"upcoming-maintenance-vms: {upc_maint_map}") + + curr_jobs = get_slurm_maintenance_job(lkp) # map job_name -> node. + log.debug(f"curr-maintenance-job-map: {curr_jobs}") + + del_jobs = set(curr_jobs.keys() - upc_maint_map.keys()) + create_jobs = {} + + for job_name, (node, _) in upc_maint_map.items(): + try: + enabled = lkp.node_nodeset(node).enable_opportunistic_maintenance + except Exception: + enabled = False + + if not enabled: + if job_name in curr_jobs: + del_jobs.add(job_name) + continue + + if job_name not in curr_jobs: + create_jobs[job_name] = node + + log.debug(f"del-maintenance-job: {del_jobs}") + for job_name in del_jobs: + delete_maintenance_job(job_name) + + log.debug(f"create-maintenance-job: {create_jobs}") + for job_name, node in create_jobs.items(): + create_maintenance_job(job_name, node) + + + +def sync_flex_migs(lkp: util.Lookup) -> None: + pass + + +def process_messages(lkp: util.Lookup) -> None: + try: + watch_delete_vm_op.watch_vm_delete_ops(lkp) + except: + log.exception("failed during watching delete VM operations") + + +def main(): + lkp = lookup() + if util.should_mount_slurm_bucket() and not lkp.is_controller: + return + try: + reconfigure_slurm() + except Exception: + log.exception("failed to reconfigure slurm") + if lkp.is_controller: + try: + process_messages(lkp) + except: + log.exception("failed to process messages") + + try: + sync_instances() + except Exception: + log.exception("failed to sync instances") + + try: + sync_flex_migs(lkp) + except Exception: + log.exception("failed to sync DWS Flex MIGs") + + try: + sync_placement_groups() + except Exception: + log.exception("failed to sync placement groups") + + try: + update_topology(lkp) + except Exception: + log.exception("failed to update topology") + + try: + sync_maintenance_reservation(lkp) + except Exception: + log.exception("failed to sync slurm reservation for scheduled maintenance") + + try: + sync_opportunistic_maintenance(lkp) + except Exception: + log.exception("failed to sync opportunistic reservation for scheduled maintenance") + + + try: + # TODO: it performs 1 to 4 GCS list requests, + # use cached version, combine with `_list_config_blobs` + install_custom_scripts(check_hash=True) + except Exception: + log.exception("failed to sync custom scripts") + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + _ = util.init_log_and_parse(parser) + + pid_file = (Path("/tmp") / Path(__file__).name).with_suffix(".pid") + with pid_file.open("w") as fp: + try: + fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB) + main() + except BlockingIOError: + sys.exit(0) diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py new file mode 100644 index 0000000000..ae36c54222 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py @@ -0,0 +1,171 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This script sorts nodes based on their `physicalHost`. + +See https://cloud.google.com/compute/docs/instances/use-compact-placement-policies + +You can reduce latency in tightly coupled HPC workloads (including distributed ML training) +by deploying them to machines that are located close together. +For example, if you deploy your workload on a single physical rack, you can expect lower latency +than if your workload is spread across multiple racks. +Sending data across multiple rack requires sending data through additional network switches. + +Example usage: +``` my_sbatch.sh +#SBATCH --ntasks-per-node=8 +#SBATCH --nodes=64 + +export SLURM_HOSTFILE=$(sort_nodes.py) + +srun -l hostname | sort +``` +""" +import os +import subprocess +import uuid +from typing import List, Optional, Dict +from collections import OrderedDict + +def order(paths: List[List[str]]) -> List[str]: + """ + Orders the leaves of the tree in a way that minimizes the sum of distance in between + each pair of neighboring nodes in the resulting order. + The resulting order will always start from the first node in the input list. + The ordering is "stable" with respect to the input order of the leaves i.e. + given a choice between two nodes (identical in other ways) it will select "nodelist-smallest" one. + + Returns a list of nodenames, ordered as described above. + """ + if not paths: return [] + class Vert: + "Represents a vertex in a *network* tree." + def __init__(self, name: str, parent: Optional["Vert"]): + self.name = name + self.parent = parent + # Use `OrderedDict` to preserve insertion order + # TODO: once we move to Python 3.7+ use regular `dict` since it has the same guarantee + self.children: OrderedDict = OrderedDict() + + # build a tree, children are ordered by insertion order + root = Vert("", None) + for path in paths: + n = root + for v in path: + if v not in n.children: + n.children[v] = Vert(v, n) + n = n.children[v] + + # walk the tree in insertion order, gather leaves + result = [] + def gather_nodes(v: Vert) -> None: + if not v.children: # this is a Slurm node + result.append(v.name) + for u in v.children.values(): + gather_nodes(u) + gather_nodes(root) + return result + + +class Instance: + def __init__(self, name: str, zone: str, physical_host: Optional[str]): + self.name = name + self.zone = zone + self.physical_host = physical_host + + +def make_path(node_name: str, inst: Optional[Instance]) -> List[str]: + if not inst: # node with unknown instance (e.g. hybrid cluster) + return ["unknown", node_name] + zone = f"zone_{inst.zone}" + if not inst.physical_host: # node without physical host info (e.g. no placement policy) + return [zone, "unknown", node_name] + + assert inst.physical_host.startswith("/"), f"Unexpected physicalHost: {inst.physical_host}" + parts = inst.physical_host[1:].split("/") + if len(parts) >= 4: + return [*parts, node_name] + return [zone, *parts, node_name] + + +def to_hostnames(nodelist: str) -> List[str]: + cmd = ["scontrol", "show", "hostnames", nodelist] + out = subprocess.run(cmd, check=True, stdout=subprocess.PIPE).stdout + return [n.decode("utf-8") for n in out.splitlines()] + + +def get_instances(node_names: List[str]) -> Dict[str, Optional[Instance]]: + fmt = ( + "--format=csv[no-heading,separator=','](zone,resourceStatus.physicalHost,name)" + ) + cmd = ["gcloud", "compute", "instances", "list", fmt] + + scp = os.path.commonprefix(node_names) + if scp: + cmd.append(f"--filter=name~'{scp}.*'") + out = subprocess.run(cmd, check=True, stdout=subprocess.PIPE).stdout + d = {} + for line in out.splitlines(): + zone, physical_host, name = line.decode("utf-8").split(",") + d[name] = Instance(name, zone, physical_host) + return {n: d.get(n) for n in node_names} + + +def main(args) -> None: + nodelist = args.nodelist or os.getenv("SLURM_NODELIST") + if not nodelist: + raise ValueError("nodelist is not provided and SLURM_NODELIST is not set") + + if args.ntasks_per_node is None: + args.ntasks_per_node = int(os.getenv("SLURM_NTASKS_PER_NODE", "") or 1) + assert args.ntasks_per_node > 0 + + output = args.output or f"hosts.{uuid.uuid4()}" + + node_names = to_hostnames(nodelist) + instannces = get_instances(node_names) + paths = [make_path(n, instannces[n]) for n in node_names] + ordered = order(paths) + + with open(output, "w") as f: + for node in ordered: + for _ in range(args.ntasks_per_node): + f.write(node) + f.write("\n") + print(output) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + "--nodelist", + type=str, + help="Slurm 'hostlist expression' of nodes to sort, if not set the value of SLURM_NODELIST environment variable will be used", + ) + parser.add_argument( + "--ntasks-per-node", + type=int, + help="""Number of times to repeat each node in resulting sorted list. +If not set, the value of SLURM_NTASKS_PER_NODE environment variable will be used, +if neither is set, defaults to 1""", + ) + parser.add_argument( + "--output", type=str, help="Output file to write, defaults to 'hosts.'" + ) + args = parser.parse_args() + main(args) diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py new file mode 100644 index 0000000000..ecef70f1cc --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py @@ -0,0 +1,126 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Any +import argparse +import logging + +import util +from util import ( + log_api_request, + batch_execute, + to_hostlist, + separate, +) +from util import lookup +import tpu +import mig_flex +import watch_delete_vm_op + +log = logging.getLogger() + +TOT_REQ_CNT = 1000 + + +def truncate_iter(iterable, max_count): + end = "..." + _iter = iter(iterable) + for i, el in enumerate(_iter, start=1): + if i >= max_count: + yield end + break + yield el + + +def delete_instance_request(name: str) -> Any: + inst = lookup().instance(name) + assert inst + + request = lookup().compute.instances().delete( + project=lookup().project, + zone=inst.zone, + instance=name, + ) + log_api_request(request) + return request + + +def delete_instances(instances): + """delete instances individually""" + invalid, valid = separate(lambda inst: bool(lookup().instance(inst)), instances) + if len(invalid) > 0: + log.debug("instances do not exist: {}".format(",".join(invalid))) + if len(valid) == 0: + log.debug("No instances to delete") + return + + requests = {inst: delete_instance_request(inst) for inst in valid} + + log.info(f"to delete {len(valid)} instances ({to_hostlist(valid)})") + ops, failed = batch_execute(requests) + for node, (_, err) in failed.items(): + log.error(f"instance {node} failed to delete: {err}") + + log.info(f"deleting {len(ops)} instances {to_hostlist(ops.keys())}") + + topic = watch_delete_vm_op.watch_delete_vm_op_topic() + for node, op in ops.items(): + topic.publish(op, node) + + + + +def suspend_nodes(nodes: List[str]) -> None: + lkp = lookup() + other_nodes, tpu_nodes = util.separate(lkp.node_is_tpu, nodes) + bulk_nodes, flex_nodes = util.separate(lkp.is_flex_node, other_nodes) + + mig_flex.suspend_flex_nodes(flex_nodes, lkp) + delete_instances(bulk_nodes) + tpu.delete_tpu_instances(tpu_nodes) + + +def main(nodelist): + """main called when run as script""" + log.debug(f"SuspendProgram {nodelist}") + + # Filter out nodes not in config.yaml + other_nodes, pm_nodes = separate( + lookup().is_power_managed_node, util.to_hostnames(nodelist) + ) + if other_nodes: + log.debug( + f"Ignoring non-power-managed nodes '{to_hostlist(other_nodes)}' from '{nodelist}'" + ) + if pm_nodes: + log.debug(f"Suspending nodes '{to_hostlist(pm_nodes)}' from '{nodelist}'") + else: + log.debug("No cloud nodes to suspend") + return + + log.info(f"suspend {nodelist}") + suspend_nodes(pm_nodes) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("nodelist", help="list of nodes to suspend") + args = util.init_log_and_parse(parser) + + main(args.nodelist) diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh new file mode 100644 index 0000000000..9079e4e4b0 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +PYTHON_SCRIPT="${SCRIPT_DIR}/suspend.py" + +# Capture all arguments passed by Slurm (the nodelist). +ALL_ARGS=("$@") + +"${PYTHON_SCRIPT}" "${ALL_ARGS[@]}" & +disown + +exit 0 diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py new file mode 100644 index 0000000000..ea4b07c64d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py @@ -0,0 +1,115 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Any +import sys +from dataclasses import dataclass, field +from datetime import datetime + +SCRIPTS_DIR = "community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts" +if SCRIPTS_DIR not in sys.path: + sys.path.append(SCRIPTS_DIR) # TODO: make this more robust + +import util + + +SOME_TS = datetime.fromisoformat("2018-09-03T20:56:35.450686+00:00") +# TODO: use "real" classes once they are defined (instead of NSDict) + +@dataclass +class Placeholder: + pass + +@dataclass +class TstNodeset: + nodeset_name: str = "cantor" + node_count_static: int = 0 + node_count_dynamic_max: int = 0 + node_conf: dict[str, Any] = field(default_factory=dict) + instance_template: Optional[str] = None + reservation_name: Optional[str] = "" + zone_policy_allow: Optional[list[str]] = field(default_factory=list) + enable_placement: bool = True + placement_max_distance: Optional[int] = None + future_reservation: Optional[str] = "" + +@dataclass +class TstPartition: + partition_name: str = "euler" + partition_nodeset: list[str] = field(default_factory=list) + partition_nodeset_tpu: list[str] = field(default_factory=list) + enable_job_exclusive: bool = False + +@dataclass +class TstCfg: + slurm_cluster_name: str = "m22" + cloud_parameters: dict[str, Any] = field(default_factory=dict) + + partitions: dict[str, TstPartition] = field(default_factory=dict) + nodeset: dict[str, TstNodeset] = field(default_factory=dict) + nodeset_tpu: dict[str, TstNodeset] = field(default_factory=dict) + nodeset_dyn: dict[str, TstNodeset] = field(default_factory=dict) + + install_dir: Optional[str] = None + output_dir: Optional[str] = None + + prolog_scripts: Optional[list[Placeholder]] = field(default_factory=list) + epilog_scripts: Optional[list[Placeholder]] = field(default_factory=list) + task_prolog_scripts: Optional[list[Placeholder]] = field(default_factory=list) + task_epilog_scripts: Optional[list[Placeholder]] = field(default_factory=list) + + +@dataclass +class TstTPU: # to prevent client initialization durint "TPU.__init__" + vmcount: int + +@dataclass +class TstMachineConf: + cpus: int + memory: int + sockets: int + sockets_per_board: int + cores_per_socket: int + boards: int + threads_per_core: int + + +@dataclass +class TstTemplateInfo: + gpu: Optional[util.AcceleratorInfo] + +def tstInstance(name: str, physical_host: Optional[str] = None): + return util.Instance( + name=name, + zone="anorien", + status="RUNNING", + creation_timestamp=SOME_TS, + resource_status=util.InstanceResourceStatus( + physical_host=physical_host, + upcoming_maintenance=None, + ), + scheduling=util.NSDict(), + role="compute", + metadata={}, + ) + +def make_to_hostnames_mock(tbl: Optional[dict[str, list[str]]]): + tbl = tbl or {} + + def se(k: str) -> list[str]: + if k not in tbl: + raise AssertionError(f"to_hostnames mock: unexpected nodelist: '{k}'") + return tbl[k] + + return se diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py new file mode 100644 index 0000000000..5fb4df5e82 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py @@ -0,0 +1,192 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from mock import Mock +from common import TstNodeset, TstCfg, TstMachineConf, TstTemplateInfo, Placeholder + +import addict # type: ignore +import conf +import util + + +def test_nodeset_tpu_lines(): + nodeset = TstNodeset( + "turbo", + node_count_static=2, + node_count_dynamic_max=3, + node_conf={"red": "velvet"}, + ) + assert conf.nodeset_tpu_lines(nodeset, util.Lookup(TstCfg())) == "\n".join( + [ + "NodeName=m22-turbo-[0-4] State=CLOUD red=velvet", + "NodeSet=turbo Nodes=m22-turbo-[0-4]", + ] + ) + + +def test_nodeset_lines(): + nodeset = TstNodeset( + "turbo", + node_count_static=2, + node_count_dynamic_max=3, + node_conf={"red": "velvet", "CPUs": 55}, + ) + lkp = util.Lookup(TstCfg()) + lkp.template_info = Mock(return_value=TstTemplateInfo( + gpu=util.AcceleratorInfo(type="Popov", count=33) + )) + mc = TstMachineConf( + cpus=5, + memory=6, + sockets=7, + sockets_per_board=8, + boards=9, + threads_per_core=10, + cores_per_socket=11, + ) + lkp.template_machine_conf = Mock(return_value=mc) # type: ignore[method-assign] + assert conf.nodeset_lines(nodeset, lkp) == "\n".join( + [ + "NodeName=m22-turbo-[0-4] State=CLOUD RealMemory=6 Boards=9 SocketsPerBoard=8 CoresPerSocket=11 ThreadsPerCore=10 CPUs=55 Gres=gpu:33 red=velvet", + "NodeSet=turbo Nodes=m22-turbo-[0-4]", + ] + ) + + +@pytest.mark.parametrize( + "value,want", + [ + ({"a": 1}, "a=1"), + ({"a": "two"}, "a=two"), + ({"a": [3, 4]}, "a=3,4"), + ({"a": ["five", "six"]}, "a=five,six"), + ({"a": None}, ""), + ({"a": ["seven", None, 8]}, "a=seven,8"), + ({"a": 1, "b": "two"}, "a=1 b=two"), + ({"a": 1, "b": None, "c": "three"}, "a=1 c=three"), + ({"a": 0, "b": None, "c": 0.0, "e": ""}, "a=0 c=0.0"), + ({"a": [0, 0.0, None, "X", "", "Y"]}, "a=0,0.0,X,,Y"), + ]) +def test_dict_to_conf(value: dict, want: str): + assert conf.dict_to_conf(value) == want + + + +@pytest.mark.parametrize( + "cfg,want", + [ + (TstCfg( + install_dir="ukulele", + ), + """LaunchParameters=enable_nss_slurm,use_interactive_step +SlurmctldParameters=cloud_dns,enable_configless,idle_on_node_suspend +SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation +ResumeProgram=ukulele/resume_wrapper.sh +ResumeFailProgram=ukulele/suspend_wrapper.sh +ResumeRate=0 +ResumeTimeout=300 +SuspendProgram=ukulele/suspend_wrapper.sh +SuspendRate=0 +SuspendTimeout=300 +TreeWidth=128 +TopologyPlugin=topology/tree +TopologyParam=SwitchAsNodeRank"""), + (TstCfg( + install_dir="ukulele", + cloud_parameters={ + "no_comma_params": True, + "private_data": None, + "scheduler_parameters": None, + "resume_rate": None, + "resume_timeout": None, + "suspend_rate": None, + "suspend_timeout": None, + "topology_plugin": None, + "topology_param": None, + "tree_width": None, + }, + ), + """SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation +ResumeProgram=ukulele/resume_wrapper.sh +ResumeFailProgram=ukulele/suspend_wrapper.sh +ResumeRate=0 +ResumeTimeout=300 +SuspendProgram=ukulele/suspend_wrapper.sh +SuspendRate=0 +SuspendTimeout=300 +TreeWidth=128 +TopologyPlugin=topology/tree +TopologyParam=SwitchAsNodeRank"""), + (TstCfg( + install_dir="ukulele", + cloud_parameters={ + "no_comma_params": True, + "private_data": [ + "events", + "jobs", + ], + "scheduler_parameters": [ + "bf_busy_nodes", + "bf_continue", + "ignore_prefer_validation", + "nohold_on_prolog_fail", + ], + "resume_rate": 1, + "resume_timeout": 2, + "suspend_rate": 3, + "suspend_timeout": 4, + "topology_plugin": "guess", + "topology_param": "yellow", + "tree_width": 5, + }, + ), + """PrivateData=events,jobs +SchedulerParameters=bf_busy_nodes,bf_continue,ignore_prefer_validation,nohold_on_prolog_fail +ResumeProgram=ukulele/resume_wrapper.sh +ResumeFailProgram=ukulele/suspend_wrapper.sh +ResumeRate=1 +ResumeTimeout=2 +SuspendProgram=ukulele/suspend_wrapper.sh +SuspendRate=3 +SuspendTimeout=4 +TreeWidth=5 +TopologyPlugin=guess +TopologyParam=yellow"""), + (TstCfg( + install_dir="ukulele", + task_prolog_scripts=[Placeholder()], + task_epilog_scripts=[Placeholder()], + ), + """LaunchParameters=enable_nss_slurm,use_interactive_step +SlurmctldParameters=cloud_dns,enable_configless,idle_on_node_suspend +TaskProlog=/slurm/custom_scripts/task_prolog.d/task-prolog +TaskEpilog=/slurm/custom_scripts/task_epilog.d/task-epilog +SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation +ResumeProgram=ukulele/resume_wrapper.sh +ResumeFailProgram=ukulele/suspend_wrapper.sh +ResumeRate=0 +ResumeTimeout=300 +SuspendProgram=ukulele/suspend_wrapper.sh +SuspendRate=0 +SuspendTimeout=300 +TreeWidth=128 +TopologyPlugin=topology/tree +TopologyParam=SwitchAsNodeRank"""), + ]) +def test_conflines(cfg, want): + assert conf.conflines(util.Lookup(cfg)) == want + + cfg.cloud_parameters = addict.Dict(cfg.cloud_parameters) + assert conf.conflines(util.Lookup(cfg)) == want diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py new file mode 100644 index 0000000000..77f1229605 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py @@ -0,0 +1,175 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import os +import pytest +import unittest.mock +import unittest +import tempfile + +from common import TstCfg, TstNodeset, TstPartition, TstTPU # needed to import util +import util +import resume +from resume import ResumeData, ResumeJobData, BulkChunk, PlacementAndNodes + +def test_get_resume_file_data_no_env(): + with unittest.mock.patch.dict(os.environ, {"SLURM_RESUME_FILE": ""}): + assert resume.get_resume_file_data() is None + + +def test_get_resume_file_data(): + with tempfile.NamedTemporaryFile() as f: + f.write(b"""{ + "jobs": [ + { + "extra": null, + "job_id": 1, + "features": null, + "nodes_alloc": "green-[0-2]", + "nodes_resume": "green-[0-1]", + "oversubscribe": "OK", + "partition": "red", + "reservation": null + } + ], + "all_nodes_resume": "green-[0-1]" +}""") + f.flush() + with ( + unittest.mock.patch.dict(os.environ, {"SLURM_RESUME_FILE": f.name}), + unittest.mock.patch("util.to_hostnames") as mock_to_hostnames, + ): + mock_to_hostnames.return_value = ["green-0", "green-1", "green-2"] + assert resume.get_resume_file_data() == ResumeData(jobs=[ + ResumeJobData( + job_id = 1, + partition="red", + nodes_alloc=["green-0", "green-1", "green-2"], + ) + ]) + mock_to_hostnames.assert_called_once_with("green-[0-2]") + + +@unittest.mock.patch("tpu.TPU.make") +@unittest.mock.patch("resume.create_placements") +def test_group_nodes_bulk(mock_create_placements, mock_tpu): + cfg = TstCfg( + nodeset={ + "n": TstNodeset(nodeset_name="n"), + }, + nodeset_tpu={ + "t": TstNodeset(nodeset_name="t"), + }, + partitions={ + "p1": TstPartition( + partition_name="p1", + enable_job_exclusive=True, + ), + "p2": TstPartition( + partition_name="p2", + partition_nodeset_tpu=["t"], + enable_job_exclusive=True, + ) + } + ) + lkp = util.Lookup(cfg) + + def mock_create_placements_se(nodes, excl_job_id, lkp): + args = (set(nodes), excl_job_id) + if ({'c-n-1', 'c-n-2', 'c-t-8', 'c-t-9'}, None) == args: + return [ + PlacementAndNodes("g0", ["c-n-1", "c-n-2"]), + PlacementAndNodes(None, ['c-t-8', 'c-t-9']), + ] + if ({"c-n-0", "c-n-8"}, 1) == args: + return [ + PlacementAndNodes("g10", ["c-n-0"]), + PlacementAndNodes("g11", ["c-n-8"]), + ] + if ({'c-t-0', 'c-t-1', 'c-t-2', 'c-t-3', 'c-t-4', 'c-t-5'}, 2) == args: + return [ + PlacementAndNodes(None, ['c-t-0', 'c-t-1', 'c-t-2', 'c-t-3', 'c-t-4', 'c-t-5']) + ] + raise AssertionError(f"unexpected invocation: '{args}'") + mock_create_placements.side_effect = mock_create_placements_se + + def mock_tpu_se(ns: str, lkp) -> TstTPU: + if ns == "t": + return TstTPU(vmcount=2) + raise AssertionError(f"unexpected invocation: '{ns}'") + mock_tpu.side_effect = mock_tpu_se + + got = resume.group_nodes_bulk( + ["c-n-0", "c-n-1", "c-n-2", "c-t-0", "c-t-1", "c-t-2", "c-t-3", "c-t-8", "c-t-9"], + ResumeData(jobs=[ + ResumeJobData(job_id=1, partition="p1", nodes_alloc=["c-n-0", "c-n-8"]), + ResumeJobData(job_id=2, partition="p2", nodes_alloc=["c-t-0", "c-t-1", "c-t-2", "c-t-3", "c-t-4", "c-t-5"]), + ]), lkp) + mock_create_placements.assert_called() + assert got == { + "c-n:jobNone:g0:0": BulkChunk( + nodes=["c-n-1", "c-n-2"], prefix="c-n", chunk_idx=0, excl_job_id=None, placement_group="g0"), + "c-n:job1:g10:0": BulkChunk( + nodes=["c-n-0"], prefix="c-n", chunk_idx=0, excl_job_id=1, placement_group="g10"), + "c-t:0": BulkChunk( + nodes=["c-t-8", "c-t-9"], prefix="c-t", chunk_idx=0, excl_job_id=None, placement_group=None), + "c-t:job2:0": BulkChunk( + nodes=["c-t-0", "c-t-1"], prefix="c-t", chunk_idx=0, excl_job_id=2, placement_group=None), + "c-t:job2:1": BulkChunk( + nodes=["c-t-2", "c-t-3"], prefix="c-t", chunk_idx=1, excl_job_id=2, placement_group=None), + } + + +@pytest.mark.parametrize( + "nodes,excl_job_id,expected", + [ + ( # TPU - no placements + ["c-t-0", "c-t-2"], 4, [PlacementAndNodes(None, ["c-t-0", "c-t-2"])] + ), + ( # disabled placements - no placemens + ["c-x-0", "c-x-2"], 4, [PlacementAndNodes(None, ["c-x-0", "c-x-2"])] + ), + ( # excl_job + ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"], 4, [ + PlacementAndNodes("c-slurmgcp-managed-n-4-0", ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"]) + ] + ), + ( # no excl_job + ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"], None, [ + PlacementAndNodes("c-slurmgcp-managed-n-0-0", ["c-n-0", "c-n-2"]), + PlacementAndNodes('c-slurmgcp-managed-n-0-1', ['c-n-2011']), + PlacementAndNodes(None, ["c-n-uno"]), + ] + ), + ], +) +def test_allocate_nodes_to_placements(nodes: list[str], excl_job_id: Optional[int], expected: list[PlacementAndNodes]): + cfg = TstCfg( + slurm_cluster_name="c", + nodeset={ + "n": TstNodeset(nodeset_name="n", enable_placement=True), + "x": TstNodeset(nodeset_name="x", enable_placement=False) + }, + nodeset_tpu={ + "t": TstNodeset(nodeset_name="t") + }) + lkp = util.Lookup(cfg) + + with unittest.mock.patch("resume.valid_placement_node") as mock_valid_placement_node: + mock_valid_placement_node.return_value = True + lkp.template_info = unittest.mock.Mock(return_value=unittest.mock.Mock(machine_type=unittest.mock.Mock(family="n1"))) + + assert resume._allocate_nodes_to_placements(nodes, excl_job_id, lkp) == expected diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py new file mode 100644 index 0000000000..df9f3a0137 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py @@ -0,0 +1,215 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import json +import mock +from pytest_unordered import unordered +from common import TstCfg, TstNodeset, TstTPU, tstInstance +import sort_nodes + +import util +import conf +import tempfile + +PRELUDE = """ +# Warning: +# This file is managed by a script. Manual modifications will be overwritten. + +""" + +def test_gen_topology_conf_empty(): + out_dir = tempfile.mkdtemp() + cfg = TstCfg(output_dir=out_dir) + conf.gen_topology_conf(util.Lookup(cfg)) + assert open(out_dir + "/cloud_topology.conf").read() == PRELUDE + "\n" + + +@mock.patch("tpu.TPU.make") +def test_gen_topology_conf(tpu_mock): + output_dir = tempfile.mkdtemp() + cfg = TstCfg( + nodeset_tpu={ + "a": TstNodeset("bold", node_count_static=4, node_count_dynamic_max=5), + "b": TstNodeset("slim", node_count_dynamic_max=3), + }, + nodeset={ + "c": TstNodeset("green", node_count_static=2, node_count_dynamic_max=3), + "d": TstNodeset("blue", node_count_static=7), + "e": TstNodeset("pink", node_count_dynamic_max=4), + }, + output_dir=output_dir, + ) + + def tpu_se(ns: str, lkp) -> TstTPU: + if ns == "bold": + return TstTPU(vmcount=3) + if ns == "slim": + return TstTPU(vmcount=1) + raise AssertionError(f"unexpected TPU name: '{ns}'") + + tpu_mock.side_effect = tpu_se + + lkp = util.Lookup(cfg) + lkp.instances = lambda: { n.name: n for n in [ # type: ignore[assignment] + # nodeset blue + tstInstance("m22-blue-0"), # no physicalHost + tstInstance("m22-blue-0", physical_host="/a/a/a"), + tstInstance("m22-blue-1", physical_host="/a/a/b"), + tstInstance("m22-blue-2", physical_host="/a/b/a"), + tstInstance("m22-blue-3", physical_host="/b/a/a"), + # nodeset green + tstInstance("m22-green-3", physical_host="/a/a/c"), + ]} + + uncompressed = conf.gen_topology(lkp) + want_uncompressed = [ + #NOTE: the switch names are not unique, it's not valid content for topology.conf + # The uniquefication and compression of names are done in the compress() method + "SwitchName=slurm-root Switches=a,b,ns_blue,ns_green,ns_pink", + # "physical" topology + 'SwitchName=a Switches=a,b', + 'SwitchName=a Nodes=m22-blue-[0-1],m22-green-3', + 'SwitchName=b Nodes=m22-blue-2', + 'SwitchName=b Switches=a', + 'SwitchName=a Nodes=m22-blue-3', + # topology "by nodeset" + "SwitchName=ns_blue Nodes=m22-blue-[4-6]", + "SwitchName=ns_green Nodes=m22-green-[0-2,4]", + "SwitchName=ns_pink Nodes=m22-pink-[0-3]", + # TPU topology + "SwitchName=tpu-root Switches=ns_bold,ns_slim", + "SwitchName=ns_bold Switches=bold-[0-3]", + "SwitchName=bold-0 Nodes=m22-bold-[0-2]", + "SwitchName=bold-1 Nodes=m22-bold-3", + "SwitchName=bold-2 Nodes=m22-bold-[4-6]", + "SwitchName=bold-3 Nodes=m22-bold-[7-8]", + "SwitchName=ns_slim Nodes=m22-slim-[0-2]"] + assert list(uncompressed.render_conf_lines()) == want_uncompressed + + compressed = uncompressed.compress() + want_compressed = [ + "SwitchName=s0 Switches=s0_[0-4]", # root + # "physical" topology + 'SwitchName=s0_0 Switches=s0_0_[0-1]', # /a + 'SwitchName=s0_0_0 Nodes=m22-blue-[0-1],m22-green-3', # /a/a + 'SwitchName=s0_0_1 Nodes=m22-blue-2', # /a/b + 'SwitchName=s0_1 Switches=s0_1_0', # /b + 'SwitchName=s0_1_0 Nodes=m22-blue-3', # /b/a + # topology "by nodeset" + "SwitchName=s0_2 Nodes=m22-blue-[4-6]", + "SwitchName=s0_3 Nodes=m22-green-[0-2,4]", + "SwitchName=s0_4 Nodes=m22-pink-[0-3]", + # TPU topology + "SwitchName=s1 Switches=s1_[0-1]", + "SwitchName=s1_0 Switches=s1_0_[0-3]", + "SwitchName=s1_0_0 Nodes=m22-bold-[0-2]", + "SwitchName=s1_0_1 Nodes=m22-bold-3", + "SwitchName=s1_0_2 Nodes=m22-bold-[4-6]", + "SwitchName=s1_0_3 Nodes=m22-bold-[7-8]", + "SwitchName=s1_1 Nodes=m22-slim-[0-2]"] + assert list(compressed.render_conf_lines()) == want_compressed + + upd, summary = conf.gen_topology_conf(lkp) + assert upd == True + want_written = PRELUDE + "\n".join(want_compressed) + "\n\n" + assert open(output_dir + "/cloud_topology.conf").read() == want_written + + summary.dump(lkp) + summary_got = json.loads(open(output_dir + "/cloud_topology.summary.json").read()) + + assert summary_got == { + "down_nodes": unordered( + [f"m22-blue-{i}" for i in (4,5,6)] + + [f"m22-green-{i}" for i in (0,1,2,4)] + + [f"m22-pink-{i}" for i in range(4)]), + "tpu_nodes": unordered( + [f"m22-bold-{i}" for i in range(9)] + + [f"m22-slim-{i}" for i in range(3)]), + 'physical_host': { + 'm22-blue-0': '/a/a/a', + 'm22-blue-1': '/a/a/b', + 'm22-blue-2': '/a/b/a', + 'm22-blue-3': '/b/a/a', + 'm22-green-3': '/a/a/c'}, + } + + + +def test_gen_topology_conf_update(): + cfg = TstCfg( + nodeset={ + "c": TstNodeset("green", node_count_static=2), + }, + output_dir=tempfile.mkdtemp(), + ) + lkp = util.Lookup(cfg) + lkp.instances = lambda: { # type: ignore[assignment] + # no instances + } + + # initial generation - reconfigure + upd, sum = conf.gen_topology_conf(lkp) + assert upd == True + sum.dump(lkp) + + # add node: node_count_static 2 -> 3 - reconfigure + lkp.cfg.nodeset["c"].node_count_static = 3 + upd, sum = conf.gen_topology_conf(lkp) + assert upd == True + sum.dump(lkp) + + # remove node: node_count_static 3 -> 2 - no reconfigure + lkp.cfg.nodeset["c"].node_count_static = 2 + upd, sum = conf.gen_topology_conf(lkp) + assert upd == False + # don't dump + + # set empty physicalHost - no reconfigure + lkp.instances = lambda: { # type: ignore[assignment] + n.name: n for n in [tstInstance("m22-green-0", physical_host="")]} + upd, sum = conf.gen_topology_conf(lkp) + assert upd == False + # don't dump + + # set physicalHost - reconfigure + lkp.instances = lambda: { # type: ignore[assignment] + n.name: n for n in [tstInstance("m22-green-0", physical_host="/a/b/c")]} + upd, sum = conf.gen_topology_conf(lkp) + assert upd == True + sum.dump(lkp) + + # change physicalHost - reconfigure + lkp.instances = lambda: { # type: ignore[assignment] + n.name: n for n in [tstInstance("m22-green-0", physical_host="/a/b/z")]} + upd, sum = conf.gen_topology_conf(lkp) + assert upd == True + sum.dump(lkp) + + # shut down node - no reconfigure + lkp.instances = lambda: {} # type: ignore[assignment] + upd, sum = conf.gen_topology_conf(lkp) + assert upd == False + # don't dump + + +@pytest.mark.parametrize( + "paths,expected", + [ + (["z/n-0", "z/n-1", "z/n-2", "z/n-3", "z/n-4", "z/n-10"], ['n-0', 'n-1', 'n-2', 'n-3', 'n-4', 'n-10']), + (["y/n-0", "z/n-1", "x/n-2", "x/n-3", "y/n-4", "g/n-10"], ['n-0', 'n-4', 'n-1', 'n-2', 'n-3', 'n-10']), + ]) +def test_sort_nodes_order(paths: list[str], expected: list[str]) -> None: + paths_expanded = [l.split("/") for l in paths] + assert sort_nodes.order(paths_expanded) == expected diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py new file mode 100644 index 0000000000..d6b42f82e5 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py @@ -0,0 +1,586 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Type + +import pytest +from mock import Mock +from datetime import datetime, timezone, timedelta +import unittest + +from common import TstNodeset, TstCfg # needed to import util +import util +from util import NodeState, MachineType, AcceleratorInfo, UpcomingMaintenance, InstanceResourceStatus, FutureReservation, ReservationDetails +from google.api_core.client_options import ClientOptions # noqa: E402 + +# Note: need to install pytest-mock + +@pytest.mark.parametrize( + "name,expected", + [ + ( + "az-buka-23", + { + "cluster": "az", + "nodeset": "buka", + "node": "23", + "prefix": "az-buka", + "range": None, + "suffix": "23", + }, + ), + ( + "az-buka-xyzf", + { + "cluster": "az", + "nodeset": "buka", + "node": "xyzf", + "prefix": "az-buka", + "range": None, + "suffix": "xyzf", + }, + ), + ( + "az-buka-[2-3]", + { + "cluster": "az", + "nodeset": "buka", + "node": "[2-3]", + "prefix": "az-buka", + "range": "[2-3]", + "suffix": None, + }, + ), + ], +) +def test_node_desc(name, expected): + assert util.lookup()._node_desc(name) == expected + + +@pytest.mark.parametrize( + "name,expected", + [ + ("az-buka-23", 23), + ("az-buka-0", 0), + ("az-buka", Exception), + ("az-buka-xyzf", ValueError), + ("az-buka-[2-3]", ValueError), + ], +) +def test_node_index(name, expected): + if type(expected) is type and issubclass(expected, Exception): + with pytest.raises(expected): + util.lookup().node_index(name) + else: + assert util.lookup().node_index(name) == expected + + +@pytest.mark.parametrize( + "name", + [ + "az-buka", + ], +) +def test_node_desc_fail(name): + with pytest.raises(Exception): + util.lookup()._node_desc(name) + + +@pytest.mark.parametrize( + "names,expected", + [ + ("pedro,pedro-1,pedro-2,pedro-01,pedro-02", "pedro,pedro-[1-2,01-02]"), + ("pedro,,pedro-1,,pedro-2", "pedro,pedro-[1-2]"), + ("pedro-8,pedro-9,pedro-10,pedro-11", "pedro-[8-9,10-11]"), + ("pedro-08,pedro-09,pedro-10,pedro-11", "pedro-[08-11]"), + ("pedro-08,pedro-09,pedro-8,pedro-9", "pedro-[8-9,08-09]"), + ("pedro-10,pedro-08,pedro-09,pedro-8,pedro-9", "pedro-[8-9,08-10]"), + ("pedro-8,pedro-9,juan-10,juan-11", "juan-[10-11],pedro-[8-9]"), + ("az,buki,vedi", "az,buki,vedi"), + ("a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12", "a[0-9,10-12]"), + ("a0,a2,a4,a6,a7,a8,a11,a12", "a[0,2,4,6-8,11-12]"), + ("seas7-0,seas7-1", "seas7-[0-1]"), + ], +) +def test_to_hostlist(names, expected): + assert util.to_hostlist(names.split(",")) == expected + + +@pytest.mark.parametrize( + "api,ep_ver,expected", + [ + ( + util.ApiEndpoint.BQ, + "v1", + ClientOptions(api_endpoint="https://bq.googleapis.com/v1/"), + ), + ( + util.ApiEndpoint.COMPUTE, + "staging_v1", + ClientOptions(api_endpoint="https://compute.googleapis.com/staging_v1/"), + ), + ( + util.ApiEndpoint.SECRET, + "v1", + ClientOptions(api_endpoint="https://secret_manager.googleapis.com/v1/"), + ), + ( + util.ApiEndpoint.STORAGE, + "beta", + ClientOptions(api_endpoint="https://storage.googleapis.com/beta/"), + ), + ( + util.ApiEndpoint.TPU, + "alpha", + ClientOptions(api_endpoint="https://tpu.googleapis.com/alpha/"), + ), + ], +) +def test_create_client_options( + api: util.ApiEndpoint, ep_ver: str, expected: ClientOptions, mocker +): + ud_mock = mocker.patch("util.universe_domain") + ep_mock = mocker.patch("util.endpoint_version") + ud_mock.return_value = "googleapis.com" + ep_mock.return_value = ep_ver + assert util.create_client_options(api).__repr__() == expected.__repr__() + + + +@pytest.mark.parametrize( + "nodeset,err", + [ + (TstNodeset(reservation_name="projects/x/reservations/y"), AssertionError), # no zones + (TstNodeset( + reservation_name="projects/x/reservations/y", + zone_policy_allow=["eine", "zwei"]), AssertionError), # multiples zones + (TstNodeset( + reservation_name="robin", + zone_policy_allow=["eine"]), ValueError), # invalid name + (TstNodeset( + reservation_name="projects/reservations/y", + zone_policy_allow=["eine"]), ValueError), # invalid name + (TstNodeset( + reservation_name="projects/x/zones/z/reservations/y", + zone_policy_allow=["eine"]), ValueError), # invalid name + ] +) +def test_nodeset_reservation_err(nodeset, err): + lkp = util.Lookup(TstCfg()) + lkp._get_reservation = Mock() + with pytest.raises(err): + lkp.nodeset_reservation(nodeset) + lkp._get_reservation.assert_not_called() # type: ignore + +@pytest.mark.parametrize( + "nodeset,policies,expected", + [ + (TstNodeset(), [], None), # no reservation + (TstNodeset( + reservation_name="projects/bobin/reservations/robin", + zone_policy_allow=["eine"]), + [], + util.ReservationDetails( + project="bobin", + zone="eine", + name="robin", + policies=[], + deployment_type=None, + reservation_mode=None, + bulk_insert_name="projects/bobin/reservations/robin")), + (TstNodeset( + reservation_name="projects/bobin/reservations/robin", + zone_policy_allow=["eine"]), + ["seven/wanders", "five/red/apples", "yum"], + util.ReservationDetails( + project="bobin", + zone="eine", + name="robin", + policies=["wanders", "apples", "yum"], + deployment_type=None, + reservation_mode=None, + bulk_insert_name="projects/bobin/reservations/robin")), + (TstNodeset( + reservation_name="projects/bobin/reservations/robin/snek/cheese-brie-6", + zone_policy_allow=["eine"]), + [], + util.ReservationDetails( + project="bobin", + zone="eine", + name="robin", + policies=[], + deployment_type=None, + reservation_mode=None, + bulk_insert_name="projects/bobin/reservations/robin/snek/cheese-brie-6")), + + ]) + +def test_nodeset_reservation_ok(nodeset, policies, expected): + lkp = util.Lookup(TstCfg()) + lkp._get_reservation = Mock() + + if not expected: + assert lkp.nodeset_reservation(nodeset) is None + lkp._get_reservation.assert_not_called() # type: ignore + return + + lkp._get_reservation.return_value = { # type: ignore + "resourcePolicies": {i: p for i, p in enumerate(policies)}, + } + assert lkp.nodeset_reservation(nodeset) == expected + lkp._get_reservation.assert_called_once_with(expected.project, expected.zone, expected.name) # type: ignore + +@pytest.mark.parametrize( + "job_info,expected_job", + [ + ( + """JobId=123 + TimeLimit=02:00:00 + JobName=myjob + JobState=PENDING + ReqNodeList=node-[1-10]""", + util.Job( + id=123, + duration=timedelta(days=0, hours=2, minutes=0, seconds=0), + name="myjob", + job_state="PENDING", + required_nodes="node-[1-10]" + ), + ), + ( + """JobId=456 + JobName=anotherjob + JobState=PENDING + ReqNodeList=node-group1""", + util.Job( + id=456, + duration=None, + name="anotherjob", + job_state="PENDING", + required_nodes="node-group1" + ), + ), + ( + """JobId=789 + TimeLimit=00:30:00 + JobState=COMPLETED""", + util.Job( + id=789, + duration=timedelta(minutes=30), + name=None, + job_state="COMPLETED", + required_nodes=None + ), + ), + ( + """JobId=101112 + TimeLimit=1-00:30:00 + JobState=COMPLETED, + ReqNodeList=node-[1-10],grob-pop-[2,1,44-77]""", + util.Job( + id=101112, + duration=timedelta(days=1, hours=0, minutes=30, seconds=0), + name=None, + job_state="COMPLETED", + required_nodes="node-[1-10],grob-pop-[2,1,44-77]" + ), + ), + ( + """JobId=131415 + TimeLimit=1-00:30:00 + JobName=mynode-1_maintenance + JobState=COMPLETED, + ReqNodeList=node-[1-10],grob-pop-[2,1,44-77]""", + util.Job( + id=131415, + duration=timedelta(days=1, hours=0, minutes=30, seconds=0), + name="mynode-1_maintenance", + job_state="COMPLETED", + required_nodes="node-[1-10],grob-pop-[2,1,44-77]" + ), + ), + ], +) +def test_parse_job_info(job_info, expected_job): + lkp = util.Lookup(TstCfg()) + assert lkp._parse_job_info(job_info) == expected_job + + + +@pytest.mark.parametrize( + "node,state,want", + [ + ("c-n-2", NodeState("DOWN", frozenset([])), NodeState("DOWN", frozenset([]))), # happy scenario + ("c-d-vodoo", None, None), # dynamic nodeset + ("c-x-44", None, None), # unknown(removed) nodeset + ("c-n-7", None, None), # Out of bounds: c-n-[0-4] - downsized nodeset + ("c-t-7", None, None), # Out of bounds: c-t-[0-4] - downsized nodeset TPU + ("c-n-2", None, RuntimeError), # something is wrong + ("c-t-2", None, RuntimeError), # something is wrong, but TPU + + # Check boundaries match [0-5) + ("c-n-5", None, None), # out of boundaries + ("c-n-4", None, RuntimeError), # within boundaries + ]) +def test_node_state(node: str, state: Optional[NodeState], want: NodeState | None | Type[Exception]): + cfg = TstCfg( + slurm_cluster_name="c", + nodeset={ + "n": TstNodeset(node_count_static=2, node_count_dynamic_max=3)}, + nodeset_tpu={ + "t": TstNodeset(node_count_static=2, node_count_dynamic_max=3)}, + nodeset_dyn={ + "d": TstNodeset()}, + ) + lkp = util.Lookup(cfg) + lkp.slurm_nodes = lambda: {node: state} if state else {} # type: ignore[assignment] + # ... see https://github.com/python/typeshed/issues/6347 + + if type(want) is type and issubclass(want, Exception): + with pytest.raises(want): + lkp.node_state(node) + else: + assert lkp.node_state(node) == want + + + +@pytest.mark.parametrize( + "jo,want", + [ + ({ + "accelerators": [ { "guestAcceleratorCount": 1, "guestAcceleratorType": "nvidia-tesla-a100" } ], + "creationTimestamp": "1969-12-31T16:00:00.000-08:00", + "description": "Accelerator Optimized: 1 NVIDIA Tesla A100 GPU, 12 vCPUs, 85GB RAM", + "guestCpus": 12, + "id": "1000012", + "imageSpaceGb": 0, + "isSharedCpu": False, + "kind": "compute#machineType", + "maximumPersistentDisks": 128, + "maximumPersistentDisksSizeGb": "263168", + "memoryMb": 87040, + "name": "a2-highgpu-1g", + "selfLink": "https://www.googleapis.com/compute/v1/projects/io-playground/zones/us-central1-a/machineTypes/a2-highgpu-1g", + "zone": "us-central1-a" + }, MachineType( + name="a2-highgpu-1g", + guest_cpus=12, + memory_mb=87040, + accelerators=[ + AcceleratorInfo(type="nvidia-tesla-a100", count=1) + ] + )), + ({ + "architecture": "X86_64", + "creationTimestamp": "1969-12-31T16:00:00.000-08:00", + "description": "8 vCPUs, 32 GB RAM", + "guestCpus": 8, + "id": "1210008", + "imageSpaceGb": 0, + "isSharedCpu": False, + "kind": "compute#machineType", + "maximumPersistentDisks": 128, + "maximumPersistentDisksSizeGb": "263168", + "memoryMb": 32768, + "name": "t2d-standard-8", + "selfLink": "https://www.googleapis.com/compute/v1/projects/io-playground/zones/europe-north2-b/machineTypes/t2d-standard-8", + "zone": "europe-north2-b" + }, MachineType( + name="t2d-standard-8", + guest_cpus=8, + memory_mb=32768, + accelerators=[] + )), + ]) +def test_MachineType_from_json(jo: dict, want: MachineType): + assert MachineType.from_json(jo) == want + +UTC, PST = timezone.utc, timezone(timedelta(hours=-8)) + +@pytest.mark.parametrize( + "got,want", + [ + # from instance.creationTimestamp: + ("2024-11-30T12:47:51.676-08:00", datetime(2024, 11, 30, 12, 47, 51, 676000, tzinfo=PST)), + # from futureReservation.creationTimestamp + ("2024-11-05T15:23:33.702-08:00", datetime(2024, 11, 5, 15, 23, 33, 702000, tzinfo=PST)), + # from futureReservation.timeWindow.endTime + ("2025-01-15T00:00:00Z", datetime(2025, 1, 15, 0, 0, tzinfo=UTC)), + # fallback to UTC if no tz is specified + ("2025-01-15T00:00:00", datetime(2025, 1, 15, 0, 0, tzinfo=UTC)), + ]) +def test_parse_gcp_timestamp(got: str, want: datetime): + assert util.parse_gcp_timestamp(got) == want + + +@pytest.mark.parametrize( + "got,want", + [ + (None, None), + (dict( + windowStartTime="2025-01-15T00:00:00Z", + somethingToIgnore="past failures", + ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), + (dict( + startTimeWindow=dict( + earliest="2025-01-15T00:00:00Z"), + somethingToIgnore="past failures", + ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), + (dict( + windowStartTime="2025-01-15T00:00:00Z", + startTimeWindow=dict( + earliest="2025-01-25T00:00:00Z"), # ignored + somethingToIgnore="past failures", + ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), + ]) +def tests_parse_UpcomingMaintenance_OK(got: dict, want: Optional[UpcomingMaintenance]): + assert UpcomingMaintenance.from_json(got) == want + + +@pytest.mark.parametrize( + "got", + [ + {}, + dict( + windowStartTime=dict( + earliest="2025-01-15T00:00:00Z")), + ]) +def tests_parse_UpcomingMaintenance_FAIL(got: dict): + with pytest.raises(ValueError): + UpcomingMaintenance.from_json(got) + + +@pytest.mark.parametrize( + "got,want", + [ + (None, InstanceResourceStatus( + physical_host=None, + upcoming_maintenance=None)), + ({}, InstanceResourceStatus( + physical_host=None, + upcoming_maintenance=None)), + (dict( + physicalHost="/aaa/bbb/ccc"), + InstanceResourceStatus( + physical_host="/aaa/bbb/ccc", + upcoming_maintenance=None)), + (dict( # invalid upcomingMaintenance field to be ignored + physicalHost="/aaa/bbb/ccc", + upcomingMaintenance="maintenance is upon us"), + InstanceResourceStatus( + physical_host="/aaa/bbb/ccc", + upcoming_maintenance=None)), + (dict( + physicalHost="/aaa/bbb/ccc", + upcomingMaintenance=dict(windowStartTime="2025-01-15T00:00:00Z")), + InstanceResourceStatus( + physical_host="/aaa/bbb/ccc", + upcoming_maintenance=UpcomingMaintenance( + window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC)))), + ]) +def test_parse_InstanceResourceStatus(got: dict, want: Optional[InstanceResourceStatus]): + assert InstanceResourceStatus.from_json(got) == want + + +def test_future_reservation_none(): + lkp = util.Lookup(TstCfg()) + assert lkp.future_reservation(TstNodeset()) == None + + +def test_future_reservation_declined(): + lkp = util.Lookup(TstCfg()) + lkp._get_future_reservation = Mock(return_value=dict( + timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-03T23:30:00Z" }, + status = {"procurementStatus": "DECLINED"}, + reservationMode = "CALENDAR", + specificReservationRequired = True, + )) + + assert lkp.future_reservation( + TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( + project='manhattan', + zone='danger', + name='zebra', + specific=True, + start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), + end_time=datetime(2025, 2, 3, 23, 30, tzinfo=timezone.utc), + reservation_mode="CALENDAR", + active_reservation=None) + lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") + +@unittest.mock.patch('util.now', return_value=datetime(2025, 2, 13, 0, 0, tzinfo=timezone.utc)) +def test_future_reservation_active(_): + lkp = util.Lookup(TstCfg()) + lkp._get_future_reservation = Mock(return_value=dict( + timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-21T23:30:00Z" }, + status = { + "procurementStatus": "FULFILLED", + "autoCreatedReservations": [ + "https://www.googleapis.com/compute/alpha/projects/manhattan/zones/danger/reservations/melon" + ], + }, + specificReservationRequired = True, + )) + lkp._get_reservation = Mock(return_value=dict()) + + assert lkp.future_reservation( + TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( + project='manhattan', + zone='danger', + name='zebra', + specific=True, + start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), + end_time=datetime(2025, 2, 21, 23, 30, tzinfo=timezone.utc), + reservation_mode=None, + active_reservation=ReservationDetails( + project='manhattan', + zone='danger', + name='melon', + policies=[], + reservation_mode=None, + bulk_insert_name="projects/manhattan/reservations/melon", + deployment_type=None)) + + lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") + lkp._get_reservation.assert_called_once_with("manhattan", "danger", "melon") + +@unittest.mock.patch('util.now', return_value=datetime(2025, 2, 28, 0, 0, tzinfo=timezone.utc)) +def test_future_reservation_inactive(_): + lkp = util.Lookup(TstCfg()) + lkp._get_future_reservation = Mock(return_value=dict( + timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-21T23:30:00Z" }, + status = { + "procurementStatus": "FULFILLED", + "autoCreatedReservations": [ + "https://www.googleapis.com/compute/alpha/projects/manhattan/zones/danger/reservations/melon" + ], + }, + reservationMode = "DEFAULT", + specificReservationRequired = True, + )) + lkp._get_reservation = Mock() + + assert lkp.future_reservation( + TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( + project='manhattan', + zone='danger', + name='zebra', + specific=True, + start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), + end_time=datetime(2025, 2, 21, 23, 30, tzinfo=timezone.utc), + reservation_mode="DEFAULT", + active_reservation=None) + + lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") + lkp._get_reservation.assert_not_called() diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test new file mode 100644 index 0000000000..46330331d4 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test @@ -0,0 +1,129 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +LOG_FILE="/var/log/slurm/chs_health_check.log" +TMP_DCGM_OUT="/tmp/dcgm.out" +TMP_ECC_ERRORS_OUT="/tmp/ecc_errors.out" + +log_step() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" +} + +# Fail gracefully if nvidia-smi or dcgmi doesn't exist +if ! type -P nvidia-smi 1>/dev/null; then + log_step "nvidia-smi not found - this script requires nvidia-smi to function" + exit 0 +fi + +if ! type -P dcgmi 1>/dev/null; then + log_step "dcgmi not found - this script requires dcgmi to function" + exit 0 +fi + +if ! type -P nv-hostengine 1>/dev/null; then + log_step "nv-hostengine not found - this script requires nv-hostengine to function" + exit 0 +fi + +################################################### +# Disable running health checks +################################################### +# Check if the environment variable '$SLURM_JOB_EXTRA' is set and contains the +# substring 'healthchecks_prolog=off' +if [[ -n "$SLURM_JOB_EXTRA" ]]; then + log_step "Environment variable SLURM_JOB_EXTRA is set. Checking if it contains healthchecks_prolog=off." + # Check if the value of the variable matches the string "healthchecks_prolog=off" + if [[ "$SLURM_JOB_EXTRA" == *"healthchecks_prolog=off"* ]]; then + log_step "Environment variable SLURM_JOB_EXTRA matches substring healthchecks_prolog=off. Skipping health checks." + exit 0 + else + log_step "Environment variable SLURM_JOB_EXTRA does NOT match substring healthchecks_prolog=off. Attempting to run health checks." + fi +else + log_step "Environment variable SLURM_JOB_EXTRA is NOT set. Attempting to run health checks." +fi + +# Exit if GPU isn't H/B 100/200 +GPU_MODEL=$(nvidia-smi --query-gpu=name --format=csv,noheader) +if ! [[ "$GPU_MODEL" =~ [BH][1-2]00 ]]; then + log_step "No Supported GPU detected" + exit 0 +fi + +NUMGPUS=$(nvidia-smi -L | wc -l) + +# Check that all GPUs are healthy via DCGM and check for ECC errors +if [ $NUMGPUS -gt 0 ]; then + log_step "Execute DCGM health check, ECC error check, and NVLink error check for GPUs" + GPULIST=$(nvidia-smi --query-gpu=index --format=csv,noheader | tr '\n' ',' | sed 's/,$//') + rm -f $TMP_DCGM_OUT + rm -f $TMP_ECC_ERRORS_OUT + + # Run DCGM checks + START_HOSTENGINE=false + if ! pidof nv-hostengine > /dev/null; then + log_step "Starting nv-hostengine..." + nv-hostengine >> "$LOG_FILE" 2>&1 + sleep 1 # Give it a moment to start up + START_HOSTENGINE=true + fi + GROUPID=$(dcgmi group -c gpuinfo | awk '{print $NF}' | tr -d ' ') + dcgmi group -g $GROUPID -a $GPULIST >> "$LOG_FILE" 2>&1 + dcgmi diag -g $GROUPID -r 1 > "$TMP_DCGM_OUT" 2>&1 + cat "$TMP_DCGM_OUT" >> "$LOG_FILE" + dcgmi group -d $GROUPID >> "$LOG_FILE" 2>&1 + + # Terminate the host engine if it was manually started + if [ "$START_HOSTENGINE" = true ]; then + log_step "Terminating nv-hostengine..." + nv-hostengine -t >> "$LOG_FILE" 2>&1 + fi + + # Check for DCGM failures + DCGM_FAILED=0 + if grep -i fail "$TMP_DCGM_OUT" > /dev/null; then + DCGM_FAILED=1 + fi + + # Check for ECC errors + nvidia-smi --query-gpu=ecc.errors.uncorrected.volatile.total --format=csv,noheader > "$TMP_ECC_ERRORS_OUT" + cat "$TMP_ECC_ERRORS_OUT" >> "$LOG_FILE" + ECC_ERRORS=$(awk -F', ' '{sum += $2} END {print sum}' "$TMP_ECC_ERRORS_OUT") + log_step "ECC Errors: $ECC_ERRORS" + + # Check for NVLink errors + NVLINK_ERRORS=$(nvidia-smi nvlink -sc 0bz -i 0 2>/dev/null | grep -i "Error Count" | awk '{sum += $3} END {print sum}') + # Set to 0 if empty/null + NVLINK_ERRORS=${NVLINK_ERRORS:-0} + log_step "NVLink Errors: $NVLINK_ERRORS" + + if [ $DCGM_FAILED -eq 1 ] || \ + [ $ECC_ERRORS -gt 0 ] || \ + [ $NVLINK_ERRORS -gt 0 ]; then + REASON="GPU issues detected: " + if [ $DCGM_FAILED -eq 1 ]; then + REASON+="DCGM test failed, " + fi + if [ $ECC_ERRORS -gt 0 ]; then + REASON+="ECC errors found ($ECC_ERRORS double-bit errors), " + fi + if [ $NVLINK_ERRORS -gt 0 ]; then + REASON+="NVLink errors detected ($NVLINK_ERRORS errors), " + fi + REASON+="see $LOG_FILE" + log_step "$REASON" + exit 1 + fi +fi diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog new file mode 100644 index 0000000000..a22ddea9e5 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Main TaskEpilog Script +# This script executes all *.sh scripts found in /slurm/custom_scripts/task_epilog.d/ +# +# slurm.conf configuration: +# TaskEpilog=/slurm/scripts/tools/task-epilog + +# Directory containing the individual task epilog scripts +EPILOG_D_DIR="/slurm/custom_scripts/task_epilog.d" + +# --- Output Handling for TaskEpilog --- +# The stdout and stderr of this script (and the sub-scripts it calls) +# are typically captured by Slurm and written to the job's output/error file +# or a separate Slurm log, depending on configuration. +# Unlike TaskProlog, stdout is not typically parsed for special commands +# like 'export' or 'print' to affect the (now finished) task's environment. +# +# --- Error Handling --- +# If any script in EPILOG_D_DIR exits with a non-zero status, +# this main script will also exit with a non-zero status. +# Slurm will log this. Depending on Slurm's configuration, +# frequent epilog failures might lead to node issues or alerts. +set -e # Exit immediately if a command exits with a non-zero status. + +# Check if the directory exists +if [[ ! -d "$EPILOG_D_DIR" ]]; then + # Log in task stdout and exit if the directory is missing. This likely indicates a configuration error. + echo "print TaskEpilog Error: Directory '$EPILOG_D_DIR' not found. Check Slurm configuration." + exit 1 +fi + +# Find and execute all *.sh scripts in the directory +# Scripts will be executed in reverse alphabetical order of their filenames. +find "$EPILOG_D_DIR" -maxdepth 1 -type f -name "*.sh" -print0 | sort -rz | while IFS= read -r -d $'\0' script; do + if [[ -x "$script" ]]; then + # Execute the script. Its stdout will be captured by this wrapper. + # Its stderr will also be passed through. + # If a sub-script exits with an error, 'set -e' will cause this wrapper to exit. + "$script" + else + # Log in task stdout a warning if a *.sh file is found but is not executable + echo "print TaskEpilog Warning: Script '$script' is not executable and will be skipped." + fi +done + +# Check if any scripts were found and executed +if [[ $(find "$EPILOG_D_DIR" -maxdepth 1 -type f -name "*.sh" | wc -l) -eq 0 ]]; then + # Log in task stdout if no scripts were found to execute + echo "print TaskEpilog Info: No executable *.sh scripts found in $EPILOG_D_DIR." +fi + +# Exit with 0 if all scripts were successful (or no scripts to run and not treated as error) +exit 0 diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog new file mode 100644 index 0000000000..feddb23209 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Main TaskProlog Script +# This script executes all *.sh scripts found in /slurm/custom_scripts/task_prolog.d/ +# +# slurm.conf configuration: +# TaskProlog=/slurm/scripts/tools/task-prolog + +# Directory containing the individual task prolog scripts +PROLOG_D_DIR="/slurm/custom_scripts/task_prolog.d" + +# --- Output Handling for TaskProlog --- +# Slurm's TaskProlog can interpret specific stdout lines: +# - "export NAME=value" : Sets an environment variable for the task. +# - "unset NAME" : Unsets an environment variable for the task. +# - "print message" : Prints a message to the task's standard output. +# +# This wrapper script will concatenate the stdout of all sub-scripts. +# If sub-scripts need to set/unset environment variables or print messages +# for the task, they should output the appropriate "export", "unset", or "print" +# commands to their own stdout. + +# --- Error Handling --- +# If any script in PROLOG_D_DIR exits with a non-zero status, +# this main script will also exit with a non-zero status. +# This will typically cause the task to fail. +set -e # Exit immediately if a command exits with a non-zero status. + +# Check if the directory exists +if [[ ! -d "$PROLOG_D_DIR" ]]; then + # Log in task stdout and exit if the directory is missing. All jobs will be failed. + echo "print TaskProlog Error: Directory '$PROLOG_D_DIR' not found. Check Slurm configuration." + exit 1 +fi + +# Find and execute all *.sh scripts in the directory +# Scripts will be executed in reverse alphabetical order of their filenames. +find "$PROLOG_D_DIR" -maxdepth 1 -type f -name "*.sh" -print0 | sort -rz | while IFS= read -r -d $'\0' script; do + if [[ -x "$script" ]]; then + # Execute the script. Its stdout will be captured by this wrapper. + # Its stderr will also be passed through. + # If a sub-script exits with an error, 'set -e' will cause this wrapper to exit. + "$script" + else + # Log a warning in task stdout if a *.sh file is found but is not executable + echo "print TaskProlog Warning: Script '$script' is not executable and will be skipped." + fi +done + +# Check if any scripts were found and executed +if [[ $(find "$PROLOG_D_DIR" -maxdepth 1 -type f -name "*.sh" | wc -l) -eq 0 ]]; then + # Log in task stdout if no scripts were found to execute + echo "print TaskProlog Info: No executable *.sh scripts found in $PROLOG_D_DIR." +fi + +# Exit with 0 if all scripts were successful (or no scripts to run and not treated as error) +exit 0 diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py new file mode 100644 index 0000000000..531f0348dc --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py @@ -0,0 +1,331 @@ +# mypy: ignore-errors +# This implementation of TPU integration is to be deprecated + +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +import socket +import logging +from pathlib import Path +import yaml + +import util +from util import create_client_options, ApiEndpoint + +from google.cloud import tpu_v2 as tpu # noqa: E402 +import google.api_core.exceptions as gExceptions # noqa: E402 + +log = logging.getLogger() + +_tpu_cache = {} + +class TPU: + """Class for handling the TPU-vm nodes""" + + State = tpu.types.cloud_tpu.Node.State + TPUS_PER_VM = 4 + __expected_states = { + "create": State.READY, + "start": State.READY, + "stop": State.STOPPED, + } + + __tpu_version_mapping = { + "V2": tpu.AcceleratorConfig().Type.V2, + "V3": tpu.AcceleratorConfig().Type.V3, + "V4": tpu.AcceleratorConfig().Type.V4, + } + + @classmethod + def make(cls, nodeset_name: str, lkp: util.Lookup) -> "TPU": + key = (id(lkp), nodeset_name) + if key not in _tpu_cache: + nodeset = lkp.cfg.nodeset_tpu[nodeset_name] + _tpu_cache[key] = cls(nodeset, lkp) + return _tpu_cache[key] + + + def __init__(self, nodeset: object, lkp: util.Lookup): + self._nodeset = nodeset + self.lkp = lkp + self._parent = f"projects/{lkp.project}/locations/{nodeset.zone}" + co = create_client_options(ApiEndpoint.TPU) + self._client = tpu.TpuClient(client_options=co) + self.data_disks = [] + for data_disk in nodeset.data_disks: + ad = tpu.AttachedDisk() + ad.source_disk = data_disk + ad.mode = tpu.AttachedDisk.DiskMode.DISK_MODE_UNSPECIFIED + self.data_disks.append(ad) + ns_ac = nodeset.accelerator_config + if ns_ac.topology != "" and ns_ac.version != "": + ac = tpu.AcceleratorConfig() + ac.topology = ns_ac.topology + ac.type_ = self.__tpu_version_mapping[ns_ac.version] + self.ac = ac + else: + req = tpu.GetAcceleratorTypeRequest( + name=f"{self._parent}/acceleratorTypes/{nodeset.node_type}" + ) + self.ac = self._client.get_accelerator_type(req).accelerator_configs[0] + self.vmcount = self.__calc_vm_from_topology(self.ac.topology) + + @property + def nodeset(self): + return self._nodeset + + @property + def preserve_tpu(self): + return self._nodeset.preserve_tpu + + @property + def node_type(self): + return self._nodeset.node_type + + @property + def tf_version(self): + return self._nodeset.tf_version + + @property + def enable_public_ip(self): + return self._nodeset.enable_public_ip + + @property + def preemptible(self): + return self._nodeset.preemptible + + @property + def reserved(self): + return self._nodeset.reserved + + @property + def service_account(self): + return self._nodeset.service_account + + @property + def zone(self): + return self._nodeset.zone + + def check_node_type(self): + if self.node_type is None: + return False + try: + request = tpu.GetAcceleratorTypeRequest( + name=f"{self._parent}/acceleratorTypes/{self.node_type}" + ) + return self._client.get_accelerator_type(request=request) is not None + except Exception: + return False + + def check_tf_version(self): + try: + request = tpu.GetRuntimeVersionRequest( + name=f"{self._parent}/runtimeVersions/{self.tf_version}" + ) + return self._client.get_runtime_version(request=request) is not None + except Exception: + return False + + def __calc_vm_from_topology(self, topology): + topo = topology.split("x") + tot = 1 + for num in topo: + tot = tot * int(num) + return tot // self.TPUS_PER_VM + + def __check_resp(self, response, op_name): + des_state = self.__expected_states.get(op_name) + # If the state is not in the table just print the response + if des_state is None: + return False + if response.__class__.__name__ != "Node": # If the response is not a node fail + return False + if response.state == des_state: + return True + return False + + def list_nodes(self): + try: + request = tpu.ListNodesRequest(parent=self._parent) + res = self._client.list_nodes(request=request) + except gExceptions.NotFound: + res = None + return res + + def list_node_names(self): + return [node.name.split("/")[-1] for node in self.list_nodes()] + + def start_node(self, nodename): + request = tpu.StartNodeRequest(name=f"{self._parent}/nodes/{nodename}") + resp = self._client.start_node(request=request).result() + return self.__check_resp(resp, "start") + + def stop_node(self, nodename): + request = tpu.StopNodeRequest(name=f"{self._parent}/nodes/{nodename}") + resp = self._client.stop_node(request=request).result() + return self.__check_resp(resp, "stop") + + def get_node(self, nodename): + try: + request = tpu.GetNodeRequest(name=f"{self._parent}/nodes/{nodename}") + res = self._client.get_node(request=request) + except gExceptions.NotFound: + res = None + return res + + def _register_node(self, nodename, ip_addr): + dns_name = socket.getnameinfo((ip_addr, 0), 0)[0] + util.run( + f"{self.lkp.scontrol} update nodename={nodename} nodeaddr={ip_addr} nodehostname={dns_name}" + ) + + def create_node(self, nodename): + if self.vmcount > 1 and not isinstance(nodename, list): + log.error( + f"Tried to create a {self.vmcount} node TPU on nodeset {self._nodeset.nodeset_name} but only received one nodename {nodename}" + ) + return False + if self.vmcount > 1 and ( + isinstance(nodename, list) and len(nodename) != self.vmcount + ): + log.error( + f"Expected to receive a list of {self.vmcount} nodenames for TPU node creation in nodeset {self._nodeset.nodeset_name}, but received this list {nodename}" + ) + return False + + node = tpu.Node() + node.accelerator_config = self.ac + node.runtime_version = f"tpu-vm-tf-{self.tf_version}" + startup_script = """ + #!/bin/bash + echo "startup script not found > /var/log/startup_error.log" + """ + with open( + Path(self.lkp.cfg.slurm_scripts_dir or util.dirs.scripts) / "startup.sh", "r" + ) as script: + startup_script = script.read() + if isinstance(nodename, list): + node_id = nodename[0] + slurm_names = [] + wid = 0 + for node_wid in nodename: + slurm_names.append(f"WORKER_{wid}:{node_wid}") + wid += 1 + else: + node_id = nodename + slurm_names = [f"WORKER_0:{nodename}"] + node.metadata = { + "slurm_docker_image": self.nodeset.docker_image, + "startup-script": startup_script, + "slurm_instance_role": "compute", + "slurm_cluster_name": self.lkp.cfg.slurm_cluster_name, + "slurm_bucket_path": self.lkp.cfg.bucket_path, + "slurm_names": ";".join(slurm_names), + "universe_domain": util.universe_domain(), + } + node.tags = [self.lkp.cfg.slurm_cluster_name] + if self.nodeset.service_account: + node.service_account.email = self.nodeset.service_account.email + node.service_account.scope = self.nodeset.service_account.scopes + node.scheduling_config.preemptible = self.preemptible + node.scheduling_config.reserved = self.reserved + node.network_config.subnetwork = self.nodeset.subnetwork + node.network_config.enable_external_ips = self.enable_public_ip + if self.data_disks: + node.data_disks = self.data_disks + + request = tpu.CreateNodeRequest(parent=self._parent, node=node, node_id=node_id) + resp = self._client.create_node(request=request).result() + if not self.__check_resp(resp, "create"): + return False + if isinstance(nodename, list): + for node_id, net_endpoint in zip(nodename, resp.network_endpoints): + self._register_node(node_id, net_endpoint.ip_address) + else: + ip_add = resp.network_endpoints[0].ip_address + self._register_node(nodename, ip_add) + return True + + def delete_node(self, nodename): + request = tpu.DeleteNodeRequest(name=f"{self._parent}/nodes/{nodename}") + try: + resp = self._client.delete_node(request=request).result() + if resp: + return self.get_node(nodename=nodename) is None + return False + except gExceptions.NotFound: + # log only error if vmcount is 1 as for other tpu vm count, this could be "phantom" nodes + if self.vmcount == 1: + log.error(f"Tpu single node {nodename} not found") + else: + # for the TPU nodes that consist in more than one vm, only the first node of the TPU a.k.a. the master node will + # exist as real TPU nodes, so the other ones are expected to not be found, check the hostname of the node that has + # not been found, and if it ends in 0, it means that is the master node and it should have been found, and in consequence + # log an error + nodehostname = yaml.safe_load( + util.run(f"{self.lkp.scontrol} --yaml show node {nodename}").stdout.rstrip() + )["nodes"][0]["hostname"] + if nodehostname.split("-")[-1] == "0": + log.error(f"TPU master node {nodename} not found") + else: + log.info(f"Deleted TPU 'phantom' node {nodename}") + # If the node is not found it is tecnichally deleted, so return success. + return True + +def _stop_tpu(node: str) -> None: + lkp = util.lookup() + tpuobj = TPU.make(lkp.node_nodeset_name(node), lkp) + if tpuobj.nodeset.preserve_tpu and tpuobj.vmcount == 1: + log.info(f"stopping node {node}") + if tpuobj.stop_node(node): + return + log.error("Error stopping node {node} will delete instead") + log.info(f"deleting node {node}") + if not tpuobj.delete_node(node): + log.error("Error deleting node {node}") + + +def delete_tpu_instances(instances: List[str]) -> None: + util.execute_with_futures(_stop_tpu, instances) + + +def start_tpu(node: List[str]): + lkp = util.lookup() + tpuobj = TPU.make(lkp.node_nodeset_name(node[0]), lkp) + + if len(node) == 1: + node = node[0] + log.debug( + f"Will create a TPU of type {tpuobj.node_type} tf_version {tpuobj.tf_version} in zone {tpuobj.zone} with name {node}" + ) + tpunode = tpuobj.get_node(node) + if tpunode is None: + if not tpuobj.create_node(nodename=node): + log.error("Error creating tpu node {node}") + else: + if tpuobj.preserve_tpu: + if not tpuobj.start_node(nodename=node): + log.error("Error starting tpu node {node}") + else: + log.info( + f"Tpu node {node} is already created, but will not start it because nodeset does not have preserve_tpu option active." + ) + else: + log.debug( + f"Will create a multi-vm TPU of type {tpuobj.node_type} tf_version {tpuobj.tf_version} in zone {tpuobj.zone} with name {node[0]}" + ) + if not tpuobj.create_node(nodename=node): + log.error("Error creating tpu node {node}") diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py new file mode 100644 index 0000000000..dafde1fea2 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py @@ -0,0 +1,2148 @@ +#!/slurm/python/venv/bin/python3.13 + +# Copyright (C) SchedMD LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Iterable, List, Tuple, Optional, Any, Dict, Sequence, Type, Callable, Union +import argparse +import base64 +from dataclasses import dataclass, field +from datetime import timedelta, datetime, timezone +import hashlib +import inspect +import json +import logging +import logging.config +import logging.handlers +import math +import os +import re +import shlex +import shutil +import socket +import subprocess +import sys +from enum import Enum +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import contextmanager +from functools import lru_cache, reduce, wraps +from itertools import chain, islice +from pathlib import Path +from time import sleep, time + +# TODO: remove "type: ignore" once moved to newer version of libraries +from google.cloud import secretmanager +from google.cloud import storage # type: ignore + +import google.auth # type: ignore +from google.oauth2 import service_account # type: ignore +import googleapiclient.discovery # type: ignore +import google_auth_httplib2 # type: ignore +from googleapiclient.http import set_user_agent # type: ignore +from google.api_core.client_options import ClientOptions +import httplib2 + +import google.api_core.exceptions as gExceptions + +import requests as requests_lib + +import yaml +from addict import Dict as NSDict # type: ignore +import file_cache + +USER_AGENT = "Slurm_GCP_Scripts/1.5 (GPN:SchedMD)" +ENV_CONFIG_YAML = os.getenv("SLURM_CONFIG_YAML") +if ENV_CONFIG_YAML: + CONFIG_FILE = Path(ENV_CONFIG_YAML) +else: + CONFIG_FILE = Path(__file__).with_name("config.yaml") +API_REQ_LIMIT = 2000 + + +def mkdirp(path: Path) -> None: + path.mkdir(parents=True, exist_ok=True) + + +scripts_dir = next( + p for p in (Path(__file__).parent, Path("/slurm/scripts")) if p.is_dir() +) + + +# load all directories as Paths into a dict-like namespace +dirs = NSDict( + home = Path("/home"), + apps = Path("/opt/apps"), + slurm = Path("/slurm"), + scripts = scripts_dir, + custom_scripts = Path("/slurm/custom_scripts"), + munge = Path("/etc/munge"), + secdisk = Path("/mnt/disks/sec"), + log = Path("/var/log/slurm"), + slurm_bucket_mount = Path("/slurm/bucket"), +) + +slurmdirs = NSDict( + prefix = Path("/usr/local"), + etc = Path("/usr/local/etc/slurm"), + state = Path("/var/spool/slurm"), + key_distribution = Path("/slurm/key_distribution"), +) + + +# TODO: Remove this hack (relies on undocumented behavior of PyYAML) +# No need to represent NSDict and Path once we move to properly typed & serializable config. +yaml.SafeDumper.yaml_representers[ + None # type: ignore +] = lambda self, data: yaml.representer.SafeRepresenter.represent_str(self, str(data)) # type: ignore + + +class ApiEndpoint(Enum): + COMPUTE = "compute" + BQ = "bq" + STORAGE = "storage" + TPU = "tpu" + SECRET = "secret_manager" + + +@dataclass(frozen=True) +class AcceleratorInfo: + type: str + count: int + + @classmethod + def from_json(cls, jo: dict) -> "AcceleratorInfo": + return cls( + type=jo["guestAcceleratorType"], + count=jo["guestAcceleratorCount"]) + +@dataclass(frozen=True) +class MachineType: + name: str + guest_cpus: int + memory_mb: int + accelerators: List[AcceleratorInfo] + + @classmethod + def from_json(cls, jo: dict) -> "MachineType": + return cls( + name=jo["name"], + guest_cpus=jo["guestCpus"], + memory_mb=jo["memoryMb"], + accelerators=[ + AcceleratorInfo.from_json(a) for a in jo.get("accelerators", [])], + ) + + @property + def family(self) -> str: + # TODO: doesn't work with N1 custom machine types + # See https://cloud.google.com/compute/docs/instances/creating-instance-with-custom-machine-type#create + return self.name.split("-")[0] + + @property + def supports_smt(self) -> bool: + # https://cloud.google.com/compute/docs/cpu-platforms + if self.family in ("t2a", "t2d", "h3", "c4a", "h4d",): + return False + if self.guest_cpus == 1: + return False + return True + + @property + def sockets(self) -> int: + return { + "h3": 2, + "h4d": 2, + "c2d": 2 if self.guest_cpus > 56 else 1, + "a3": 2, + "c2": 2 if self.guest_cpus > 30 else 1, + "c3": 2 if self.guest_cpus > 88 else 1, + "c3d": 2 if self.guest_cpus > 180 else 1, + "c4": 2 if self.guest_cpus > 96 else 1, + "c4d": 2 if self.guest_cpus > 192 else 1, + }.get( + self.family, + 1, # assume 1 socket for all other families + ) + + +@dataclass(frozen=True) +class UpcomingMaintenance: + window_start_time: datetime + + @classmethod + def from_json(cls, jo: Optional[dict]) -> Optional["UpcomingMaintenance"]: + if jo is None: + return None + try: + if "windowStartTime" in jo: + ts = parse_gcp_timestamp(jo["windowStartTime"]) + elif "startTimeWindow" in jo: + ts = parse_gcp_timestamp(jo["startTimeWindow"]["earliest"]) + else: + raise Exception("Neither windowStartTime nor startTimeWindow are found") + except BaseException as e: + raise ValueError(f"Unexpected format for upcomingMaintenance: {jo}") from e + return cls(window_start_time=ts) + +@dataclass(frozen=True) +class InstanceResourceStatus: + physical_host: Optional[str] + upcoming_maintenance: Optional[UpcomingMaintenance] + + @classmethod + def from_json(cls, jo: Optional[dict]) -> "InstanceResourceStatus": + if not jo: + return cls( + physical_host=None, + upcoming_maintenance=None, + ) + + try: + maint = UpcomingMaintenance.from_json(jo.get("upcomingMaintenance")) + except ValueError as e: + log.exception("Failed to parse upcomingMaintenance, ignoring") + maint = None # intentionally swallow exception + + return cls( + physical_host=jo.get("physicalHost"), + upcoming_maintenance=maint, + ) + + +@dataclass(frozen=True) +class Instance: + name: str + zone: str + status: str + creation_timestamp: datetime + role: Optional[str] + resource_status: InstanceResourceStatus + metadata: Dict[str, str] + # TODO: use proper InstanceScheduling class + scheduling: NSDict + + @classmethod + def from_json(cls, jo: dict) -> "Instance": + return cls( + name=jo["name"], + zone=trim_self_link(jo["zone"]), + status=jo["status"], + creation_timestamp=parse_gcp_timestamp(jo["creationTimestamp"]), + resource_status=InstanceResourceStatus.from_json(jo.get("resourceStatus")), + scheduling=NSDict(jo.get("scheduling")), + role = jo.get("labels", {}).get("slurm_instance_role"), + metadata = {k["key"]: k["value"] for k in jo.get("metadata", {}).get("items", [])} + ) + + +@dataclass(frozen=True) +class NSMount: + server_ip: str + local_mount: Path + remote_mount: Path + fs_type: str + mount_options: str + +@lru_cache(maxsize=1) +def default_credentials(): + return google.auth.default()[0] + + +@lru_cache(maxsize=1) +def authentication_project(): + return google.auth.default()[1] + + +DEFAULT_UNIVERSE_DOMAIN = "googleapis.com" + + +def now() -> datetime: + """ + Return current time as timezone-aware datetime. + + IMPORTANT: DO NOT use `datetime.now()`, unless you explicitly need to have tz-naive datetime. + Otherwise there is a risk of getting: "cannot compare naive and aware datetimes" error, + since all timetstamps we receive from GCP API are tz-aware. + + Another motivation for this function is to allow to mock time in tests. + """ + return datetime.now(timezone.utc) + +def parse_gcp_timestamp(s: str) -> datetime: + """ + Parse timestamp strings returned by GCP API into datetime. + Works with both Zulu and non-Zulu timestamps. + NOTE: It always return tz-aware datetime (fallbacks to UTC and logs error). + """ + # Requires Python >= 3.7 + # TODO: Remove this "hack" of trimming the Z from timestamps once we move to Python 3.11 + # (context: https://discuss.python.org/t/parse-z-timezone-suffix-in-datetime/2220/30) + ts = datetime.fromisoformat(s.replace('Z', '+00:00')) + if ts.tzinfo is None: # fallback to UTC + log.error(f"Received timestamp without timezone info: {s}") + ts = ts.replace(tzinfo=timezone.utc) + return ts + + +def universe_domain() -> str: + try: + return instance_metadata("attributes/universe_domain") + except MetadataNotFoundError: + return DEFAULT_UNIVERSE_DOMAIN + + +def endpoint_version(api: ApiEndpoint) -> Optional[str]: + return lookup().endpoint_versions.get(api.value, None) + + +@lru_cache(maxsize=1) +def get_credentials() -> Optional[service_account.Credentials]: + """Get credentials for service account""" + key_path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + if key_path is not None: + credentials = service_account.Credentials.from_service_account_file( + key_path, scopes=[f"https://www.{universe_domain()}/auth/cloud-platform"] + ) + else: + credentials = default_credentials() + + return credentials + + +@lru_cache(maxsize=1) +def get_dev_key() -> Optional[str]: + """Get dev key for project (uses json or yaml format)""" + try: + with open("/etc/slurm/slurm_vars.yaml", 'r') as file: + data = yaml.safe_load(file) + return data['google_developer_key'] + except: + return None + + +def create_client_options(api: ApiEndpoint) -> ClientOptions: + """Create client options for cloud endpoints""" + ver = endpoint_version(api) + ud = universe_domain() + options = {} + if ud and ud != DEFAULT_UNIVERSE_DOMAIN: + options["universe_domain"] = ud + if ver: + options["api_endpoint"] = f"https://{api.value}.{ud}/{ver}/" + co = ClientOptions(**options) + log.debug(f"Using ClientOptions = {co} for API: {api.value}") + return co + +log = logging.getLogger() + + +def access_secret_version(project_id, secret_id, version_id="latest"): + """ + Access the payload for the given secret version if one exists. The version + can be a version number as a string (e.g. "5") or an alias (e.g. "latest"). + """ + co = create_client_options(ApiEndpoint.SECRET) + client = secretmanager.SecretManagerServiceClient(client_options=co) + name = f"projects/{project_id}/secrets/{secret_id}/versions/{version_id}" + try: + response = client.access_secret_version(request={"name": name}) + log.debug(f"Secret '{name}' was found.") + payload = response.payload.data.decode("UTF-8") + except gExceptions.NotFound: + log.debug(f"Secret '{name}' was not found!") + payload = None + + return payload + + +def parse_self_link(self_link: str): + """Parse a selfLink url, extracting all useful values + https://.../v1/projects//regions//... + {'project': , 'region': , ...} + can also extract zone, instance (name), image, etc + """ + link_patt = re.compile(r"(?P[^\/\s]+)s\/(?P[^\s\/]+)") + return NSDict(link_patt.findall(self_link)) + + +def parse_bucket_uri(uri: str): + """ + Parse a bucket url + E.g. gs:/// + """ + pattern = re.compile(r"gs://(?P[^/\s]+)/(?P([^/\s]+)(/[^/\s]+)*)") + matches = pattern.match(uri) + assert matches, f"Unexpected bucker URI: '{uri}'" + return matches.group("bucket"), matches.group("path") + + +def trim_self_link(link: str): + """get resource name from self link url, eg. + https://.../v1/projects//regions/ + -> + """ + try: + return link[link.rindex("/") + 1 :] + except ValueError: + raise Exception(f"'/' not found, not a self link: '{link}' ") + + +def execute_with_futures(func, seq): + with ThreadPoolExecutor() as exe: + futures = [] + for i in seq: + future = exe.submit(func, i) + futures.append(future) + for future in as_completed(futures): + result = future.exception() + if result is not None: + raise result + + +def map_with_futures(func, seq): + with ThreadPoolExecutor() as exe: + futures = [] + for i in seq: + future = exe.submit(func, i) + futures.append(future) + for future in futures: + # Will be result or raise Exception + res = None + try: + res = future.result() + except Exception as e: + res = e + yield res + +def should_mount_slurm_bucket() -> bool: + try: + return instance_metadata("attributes/slurm_bucket_mount", silent=True).lower() == "true" + except MetadataNotFoundError: + return False + + +def _get_bucket_and_common_prefix() -> Tuple[str, str]: + uri = instance_metadata("attributes/slurm_bucket_path") + return parse_bucket_uri(uri) + +def blob_get(file): + bucket_name, path = _get_bucket_and_common_prefix() + blob_name = f"{path}/{file}" + return storage_client().get_bucket(bucket_name).blob(blob_name) + + +def blob_list(prefix="", delimiter=None): + bucket_name, path = _get_bucket_and_common_prefix() + blob_prefix = f"{path}/{prefix}" + # Note: The call returns a response only when the iterator is consumed. + blobs = storage_client().list_blobs( + bucket_name, prefix=blob_prefix, delimiter=delimiter + ) + return [blob for blob in blobs] + +def file_list(prefix="", subpath="") -> List[os.DirEntry]: + path = dirs.slurm_bucket_mount + file_prefix = f"{path}/{subpath}" + try: + files = os.scandir(file_prefix) + return [file for file in files if file.name.startswith(prefix)] + except: + return [] + # Not considering lack of file's existence as fatal (we may check for files we know don't exist). + # Responsibility of callee to determine if it is fatal or not, blob_list returns empty iterator in similar cases. + +def hash_file(fullpath: Path) -> str: + with open(fullpath, "rb") as f: + file_hash = hashlib.md5() + chunk = f.read(8192) + while chunk: + file_hash.update(chunk) + chunk = f.read(8192) + return base64.b64encode(file_hash.digest()).decode("utf-8") + + +def install_custom_scripts(check_hash:bool=False): + """download custom scripts from gcs bucket""" + role, tokens = lookup().instance_role, [] + + mounted_scripts=False + if should_mount_slurm_bucket() and role != "controller": + mounted_scripts=True + + all_prolog_tokens = ["prolog", "epilog", "task_prolog", "task_epilog"] + if role == "controller": + tokens = ["controller"] + all_prolog_tokens + elif role == "compute": + tokens = [f"nodeset-{lookup().node_nodeset_name()}"] + all_prolog_tokens + elif role == "login": + tokens = [f"login-{instance_login_group()}"] + + prefixes = [f"slurm-{tok}-script" for tok in tokens] + + # TODO: use single `blob_list`, to reduce ~4x number of GCS requests + if mounted_scripts: + source_collection = list(chain.from_iterable(file_list(prefix=p) for p in prefixes)) + else: + source_collection = list(chain.from_iterable(blob_list(prefix=p) for p in prefixes)) + + script_pattern = re.compile(r"^slurm-(?P\S+)-script-(?P\S+)") + for source in source_collection: + if mounted_scripts: + m = script_pattern.match(source.name) + else: + m = script_pattern.match(Path(source.name).name) + + if not m: + log.warning(f"found blob that doesn't match expected pattern: {source.name}") + continue + path_parts = m["path"].split("-") + path_parts[0] += ".d" + stem, _, ext = m["name"].rpartition("_") + filename = ".".join((stem, ext)) + + path = Path(*path_parts, filename) + fullpath = (dirs.custom_scripts / path).resolve() + mkdirp(fullpath.parent) + + for par in path.parents: + chown_slurm(dirs.custom_scripts / par) + need_update = True + + if check_hash and fullpath.exists() and isinstance(source,storage.Blob): + # TODO: MD5 reported by gcloud may differ from the one calculated here (e.g. if blob got gzipped), + # consider using gCRC32C + need_update = hash_file(fullpath) != source.md5_hash + + log.info(f"installing custom script: {path} from {source.name}") + + if isinstance(source,os.DirEntry): + shutil.copy(source.path, fullpath) #Needs to be copied since mounted nfs is read-only + chown_slurm(fullpath, mode=0o755) + + elif need_update: + with fullpath.open("wb") as f: + source.download_to_file(f) + chown_slurm(fullpath, mode=0o755) + +def compute_service(version="beta"): + """Make thread-safe compute service handle + creates a new Http for each request + """ + credentials = get_credentials() + dev_key = get_dev_key() + + def build_request(http, *args, **kwargs): + new_http = set_user_agent(httplib2.Http(), USER_AGENT) + if credentials is not None: + new_http = google_auth_httplib2.AuthorizedHttp(credentials, http=new_http) + return googleapiclient.http.HttpRequest(new_http, *args, **kwargs) + + ver = endpoint_version(ApiEndpoint.COMPUTE) + disc_url = googleapiclient.discovery.DISCOVERY_URI + if ver: + version = ver + disc_url = disc_url.replace(DEFAULT_UNIVERSE_DOMAIN, universe_domain()) + + log.debug(f"Using version={version} of Google Compute Engine API") + return googleapiclient.discovery.build( + "compute", + version, + requestBuilder=build_request, + credentials=credentials, + developerKey=dev_key, + discoveryServiceUrl=disc_url, + cache_discovery=False, # See https://github.com/googleapis/google-api-python-client/issues/299 + ) + +def storage_client() -> storage.Client: + """ + Config-independent storage client + """ + ud = universe_domain() + co = {} + if ud and ud != DEFAULT_UNIVERSE_DOMAIN: + co["universe_domain"] = ud + return storage.Client(client_options=ClientOptions(**co)) + + +class DeffetiveStoredConfigError(Exception): + """ + Raised when config can not be loaded and assembled from bucket + """ + pass + + +def _fill_cfg_defaults(cfg: NSDict) -> NSDict: + if not cfg.slurm_log_dir: + cfg.slurm_log_dir = dirs.log + if not cfg.slurm_bin_dir: + cfg.slurm_bin_dir = slurmdirs.prefix / "bin" + if not cfg.slurm_control_host: + try: + control_dns_name = instance_metadata("attributes/slurm_control_dns", silent=True) + cfg.slurm_control_host = control_dns_name + except MetadataNotFoundError: + cfg.slurm_control_host = f"{cfg.slurm_cluster_name}-controller" + if not cfg.slurm_control_host_port: + cfg.slurm_control_host_port = "6820-6830" + return cfg + +@dataclass +class _ConfigBlobs: + """ + "Private" class that represent a collection of GCS blobs for configuration + """ + core: storage.Blob + controller_addr: Optional[storage.Blob] + partition: List[storage.Blob] = field(default_factory=list) + nodeset: List[storage.Blob] = field(default_factory=list) + nodeset_dyn: List[storage.Blob] = field(default_factory=list) + nodeset_tpu: List[storage.Blob] = field(default_factory=list) + login_group: List[storage.Blob] = field(default_factory=list) + + @property + def hash(self) -> str: + h = hashlib.md5() + all = [self.core] + self.partition + self.nodeset + self.nodeset_dyn + self.nodeset_tpu + if self.controller_addr: + all.append(self.controller_addr) + + # sort blobs so hash is consistent + for blob in sorted(all, key=lambda b: b.name): + h.update(blob.md5_hash.encode("utf-8")) + return h.hexdigest() + +@dataclass +class _ConfigFiles: + """ + "Private" class that represent a collection of files for configuration + """ + core: Path + controller_addr: Optional[Path] + partition: List[Path] = field(default_factory=list) + nodeset: List[Path] = field(default_factory=list) + nodeset_dyn: List[Path] = field(default_factory=list) + nodeset_tpu: List[Path] = field(default_factory=list) + login_group: List[Path] = field(default_factory=list) + +def _list_config_blobs() -> _ConfigBlobs: + _, common_prefix = _get_bucket_and_common_prefix() + + core: Optional[storage.Blob] = None + controller_addr: Optional[storage.Blob] = None + rest: Dict[str, List[storage.Blob]] = {"partition": [], "nodeset": [], "nodeset_dyn": [], "nodeset_tpu": [], "login_group": []} + + is_controller = instance_role() == "controller" + + for blob in blob_list(prefix=""): + if blob.name == f"{common_prefix}/config.yaml": + core = blob + if blob.name == f"{common_prefix}/controller_addr.yaml" and not is_controller: + # Don't add this config blobs for controller to avoid "double reconfiguration": + # Initially this file doesn't exist and produce later by `setup_controller`; + # Appearance of this blob would trigger change in combined hash of config files; + # Ignore existence of this file for controller, assume that + # no other instance nodes will proceed with configuration until this file is created. + controller_addr = blob + for key in rest.keys(): + if blob.name.startswith(f"{common_prefix}/{key}_configs/"): + rest[key].append(blob) + + if core is None: + raise DeffetiveStoredConfigError(f"{common_prefix}/config.yaml not found in bucket") + + return _ConfigBlobs(core=core, controller_addr=controller_addr, **rest) + +def _list_config_files() -> _ConfigFiles: + file_dir = dirs.slurm_bucket_mount + core: Optional[Path] = None + controller_addr: Optional[Path] = None + rest: Dict[str, List[Path]] = {"partition": [], "nodeset": [], "nodeset_dyn": [], "nodeset_tpu": [], "login_group": []} + + if Path(f"{file_dir}/config.yaml").exists(): + core = Path(f"{file_dir}/config.yaml") + + for key in rest.keys(): + for f in file_list(subpath=f"{key}_configs"): + rest[key].append(f.path) + + if core is None: + raise Exception(f"config.yaml was not found in mounted folder: {dirs.slurm_bucket_mount}") #Intentionally not using DeffetiveStoredConfigError as this is considered a fatal error + + return _ConfigFiles(core=core, controller_addr=None, **rest) + +def _fetch_config(old_hash: Optional[str]) -> Optional[Tuple[NSDict, str]]: + """Fetch config from bucket, returns None if no changes are detected.""" + blobs = _list_config_blobs() + if old_hash == blobs.hash: + return None + + def _download(bs) -> List[Any]: + return [yaml.safe_load(b.download_as_text()) for b in bs] + + return _assemble_config( + core=_download([blobs.core])[0], + controller_addr=_download([blobs.controller_addr])[0] if blobs.controller_addr else None, + partitions=_download(blobs.partition), + nodesets=_download(blobs.nodeset), + nodesets_dyn=_download(blobs.nodeset_dyn), + nodesets_tpu=_download(blobs.nodeset_tpu), + login_groups=_download(blobs.login_group), + ), blobs.hash + +def _fetch_mounted_config() -> Optional[Tuple[NSDict, str]]: + if not dirs.slurm_bucket_mount.is_mount(): + raise Exception(f"{dirs.slurm_bucket_mount} is not mounted") + + files = _list_config_files() + + def _load(files) -> List[Any]: + file_yaml=[] + for file in files: + with open(file, "r") as f: + file_yaml.append(yaml.safe_load(f)) + return file_yaml + + return _assemble_config( + core=_load([files.core])[0], + controller_addr=None, + partitions=_load(files.partition), + nodesets=_load(files.nodeset), + nodesets_dyn=_load(files.nodeset_dyn), + nodesets_tpu=_load(files.nodeset_tpu), + login_groups=_load(files.login_group), + ) + +def controller_lookup_self_ip() -> str: + assert instance_role() == "controller" + # Get IP of LAST network-interface + # TODO: Consider change order of NICs definition, so right NIC is always @0. + idx = instance_metadata("network-interfaces").split()[-1] # either `0/` or `1/` + return instance_metadata(f"network-interfaces/{idx}ip") + +def _assemble_config( + core: Any, + controller_addr: Optional[Any], + partitions: List[Any], + nodesets: List[Any], + nodesets_dyn: List[Any], + nodesets_tpu: List[Any], + login_groups: List[Any], + ) -> NSDict: + cfg = NSDict(core) + + if cfg.controller_network_attachment: + # lookup controller address + if instance_role() == "controller": + # ignore stored value of `controller_addr`, it will be overwritten during `setup_controller` + cfg.slurm_control_addr = controller_lookup_self_ip() + else: + if not controller_addr: + raise DeffetiveStoredConfigError("controller_addr.yaml not found in bucket") + cfg.slurm_control_addr = controller_addr["slurm_control_addr"] + + # add partition configs + for p_yaml in partitions: + p_cfg = NSDict(p_yaml) + assert p_cfg.get("partition_name"), "partition_name is required" + p_name = p_cfg.partition_name + assert p_name not in cfg.partitions, f"partition {p_name} already defined" + cfg.partitions[p_name] = p_cfg + + # add nodeset configs + ns_names = set() + def _add_nodesets(yamls: List[Any], target: dict): + for ns_yaml in yamls: + ns_cfg = NSDict(ns_yaml) + assert ns_cfg.get("nodeset_name"), "nodeset_name is required" + ns_name = ns_cfg.nodeset_name + assert ns_name not in ns_names, f"nodeset {ns_name} already defined" + target[ns_name] = ns_cfg + ns_names.add(ns_name) + + _add_nodesets(nodesets, cfg.nodeset) + _add_nodesets(nodesets_dyn, cfg.nodeset_dyn) + _add_nodesets(nodesets_tpu, cfg.nodeset_tpu) + + # validate that configs for all referenced nodesets are present + for p in cfg.partitions.values(): + for ns_name in chain(p.partition_nodeset, p.partition_nodeset_dyn, p.partition_nodeset_tpu): + if ns_name not in ns_names: + raise DeffetiveStoredConfigError(f"nodeset {ns_name} not defined in config") + + for lg_yaml in login_groups: + lg_cfg = NSDict(lg_yaml) + assert lg_cfg.get("group_name"), "group_name is required" + lg_name = lg_cfg.group_name + assert lg_name not in cfg.login_groups + cfg.login_groups[lg_name] = lg_cfg + + if instance_role() == "login": + group = instance_login_group() + if group not in cfg.login_groups: + raise DeffetiveStoredConfigError(f"login group '{group}' does not exist in config") + + return _fill_cfg_defaults(cfg) + +def fetch_config() -> Tuple[bool, NSDict]: + """ + Fetches config from bucket and saves it locally + Returns True if new (updated) config was fetched + """ + hash_file = Path("/slurm/scripts/.config.hash") + old_hash = hash_file.read_text() if hash_file.exists() else None + + if should_mount_slurm_bucket() and instance_role() != "controller": + cfg = _fetch_mounted_config() + CONFIG_FILE.write_text(yaml.dump(cfg, Dumper=Dumper)) + chown_slurm(CONFIG_FILE) + return False, cfg + + cfg_and_hash = _fetch_config(old_hash=old_hash) + + if not cfg_and_hash: + return False, _load_config() + + cfg, hash = cfg_and_hash + hash_file.write_text(hash) + chown_slurm(hash_file) + CONFIG_FILE.write_text(yaml.dump(cfg, Dumper=Dumper)) + chown_slurm(CONFIG_FILE) + return True, cfg + +def owned_file_handler(filename): + """create file handler""" + chown_slurm(filename) + return logging.handlers.WatchedFileHandler(filename, delay=True) + +def get_log_path() -> Path: + """ + Returns path to log file for the current script. + e.g. resume.py -> /var/log/slurm/resume.log + """ + cfg_log_dir = lookup().cfg.slurm_log_dir + log_dir = Path(cfg_log_dir) if cfg_log_dir else dirs.log + return (log_dir / Path(sys.argv[0]).name).with_suffix(".log") + +def init_log_and_parse(parser: argparse.ArgumentParser) -> argparse.Namespace: + parser.add_argument( + "--debug", + "-d", + dest="loglevel", + action="store_const", + const=logging.DEBUG, + default=logging.INFO, + help="Enable debugging output", + ) + parser.add_argument( + "--trace-api", + "-t", + action="store_true", + help="Enable detailed api request output", + ) + args = parser.parse_args() + loglevel = args.loglevel + if lookup().cfg.enable_debug_logging: + loglevel = logging.DEBUG + if args.trace_api: + lookup().cfg.extra_logging_flags["trace_api"] = True + # Configure root logger + logging.config.dictConfig({ + "version": 1, + "disable_existing_loggers": True, + "formatters": { + "standard": { + "format": "%(levelname)s: %(message)s", + }, + "stamp": { + "format": "%(asctime)s %(levelname)s: %(message)s", + }, + }, + "handlers": { + "stdout_handler": { + "level": logging.DEBUG, + "formatter": "standard", + "class": "logging.StreamHandler", + "stream": sys.stdout, + }, + "file_handler": { + "()": owned_file_handler, + "level": logging.DEBUG, + "formatter": "stamp", + "filename": get_log_path(), + }, + }, + "root": { + "handlers": ["stdout_handler", "file_handler"], + "level": loglevel, + }, + }) + + sys.excepthook = _handle_exception + + return args + + +def log_api_request(request): + """log.trace info about a compute API request""" + if not lookup().cfg.extra_logging_flags.get("trace_api"): + return + # output the whole request object as pretty yaml + # the body is nested json, so load it as well + rep = json.loads(request.to_json()) + if rep.get("body", None) is not None: + rep["body"] = json.loads(rep["body"]) + pretty_req = yaml.safe_dump(rep).rstrip() + # label log message with the calling function + log.debug(f"{inspect.stack()[1].function}:\n{pretty_req}") + + +def _handle_exception(exc_type, exc_value, exc_trace): + """log exceptions other than KeyboardInterrupt""" + if not issubclass(exc_type, KeyboardInterrupt): + log.exception("Fatal exception", exc_info=(exc_type, exc_value, exc_trace)) + sys.__excepthook__(exc_type, exc_value, exc_trace) + + +def run( + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + timeout=None, + check=True, + universal_newlines=True, + **kwargs, +): + """Wrapper for subprocess.run() with convenient defaults""" + if isinstance(args, list): + args = list(filter(lambda x: x is not None, args)) + args = " ".join(args) + if not shell and isinstance(args, str): + args = shlex.split(args) + log.debug(f"run: {args}") + try: + result = subprocess.run( + args, + stdout=stdout, + stderr=stderr, + shell=shell, + timeout=timeout, + check=check, + universal_newlines=universal_newlines, + **kwargs, + ) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + log_subprocess(e) + raise + log_subprocess(result) + return result + +def log_subprocess(subj: subprocess.CalledProcessError | subprocess.TimeoutExpired | subprocess.CompletedProcess) -> None: + match subj: + case subprocess.CompletedProcess(returncode=0): + # Do not log successful runs, to not overwhelm logs (e.g. scontrol show jobs --json) + # TODO: consider still doing it in DEBUG or trim output to few KBs. + return + case subprocess.CompletedProcess(): # non-zero returncode + log.error(f"Command '{subj.args}' returned exit status {subj.returncode}.") + case subprocess.CalledProcessError() | subprocess.TimeoutExpired(): + log.error(str(subj)) + + + def normalize(out: None | str | bytes) -> None | str: + """ + Turns stderr and stdout into string: + > A bytes sequence, or a string if run() was called with an encoding, errors, or text=True. None if was not captured. + """ + match out: + case None: + return None + case str(): + return out.strip() + case bytes(): + return out.decode().strip() + case _: + return repr(out) + + if stdout := normalize(subj.stdout): + log.error(f"stdout: {stdout}") + if stderr := normalize(subj.stderr): + log.error(f"stderr: {stderr}") + + +def chown_slurm(path: Path, mode=None) -> None: + if path.exists(): + if mode: + path.chmod(mode) + else: + mkdirp(path.parent) + if mode: + path.touch(mode=mode) + else: + path.touch() + try: + shutil.chown(path, user="slurm", group="slurm") + except LookupError: + log.warning(f"User 'slurm' does not exist. Cannot 'chown slurm:slurm {path}'.") + except PermissionError: + log.warning(f"Not authorized to 'chown slurm:slurm {path}'.") + except Exception as err: + log.error(err) + + +@contextmanager +def cd(path): + """Change working directory for context""" + prev = Path.cwd() + os.chdir(path) + try: + yield + finally: + os.chdir(prev) + + +def cached_property(f): + return property(lru_cache()(f)) + + +def retry(max_retries: int, init_wait_time: float, warn_msg: str, exc_type: Type[Exception]): + """Retries functions that raises the exception exc_type. + Retry time is increased by a factor of two for every iteration. + + Args: + max_retries (int): Maximum number of retries + init_wait_time (float): Initial wait time in secs + warn_msg (str): Message to print during retries + exc_type (Exception): Exception type to check for + """ + + if max_retries <= 0: + raise ValueError("Incorrect value for max_retries, must be >= 1") + if init_wait_time <= 0.0: + raise ValueError("Invalid value for init_wait_time, must be > 0.0") + + def decorator(f): + @wraps(f) + def wrapper(*args, **kwargs): + retry = 0 + secs = init_wait_time + captured_exc: Optional[BaseException] = None + while retry < max_retries: + try: + return f(*args, **kwargs) + except exc_type as e: + captured_exc = e + log.warn(f"{warn_msg}, retrying in {secs}") + sleep(secs) + retry += 1 + secs *= 2 + assert captured_exc + raise captured_exc + + return wrapper + + return decorator + + +def separate(pred: Callable[[Any], bool], coll: Iterable[Any]) -> Tuple[List[Any], List[Any]]: + """filter into 2 lists based on pred returning True or False + returns ([False], [True]) + """ + res: Tuple[List[Any], List[Any]] = ([],[]) + for el in coll: + res[pred(el)].append(el) + return res + + +def chunked(iterable, n=API_REQ_LIMIT): + """group iterator into chunks of max size n""" + it = iter(iterable) + while True: + chunk = list(islice(it, n)) + if not chunk: + return + yield chunk + +def groupby_unsorted(seq: Sequence[Any], key): + indices = defaultdict(list) + for i, el in enumerate(seq): + indices[key(el)].append(i) + for k, idxs in indices.items(): + yield k, (seq[i] for i in idxs) + + +@lru_cache(maxsize=32) +def find_ratio(a, n, s, r0=None): + """given the start (a), count (n), and sum (s), find the ratio required""" + if n == 2: + return s / a - 1 + an = a * n + if n == 1 or s == an: + return 1 + if r0 is None: + # we only need to know which side of 1 to guess, and the iteration will work + r0 = 1.1 if an < s else 0.9 + + # geometric sum formula + def f(r): + return a * (1 - r**n) / (1 - r) - s + + # derivative of f + def df(r): + rm1 = r - 1 + rn = r**n + return (a * (rn * (n * rm1 - r) + r)) / (r * rm1**2) + + MIN_DR = 0.0001 # negligible change + r = r0 + # print(f"r(0)={r0}") + MAX_TRIES = 64 + for i in range(1, MAX_TRIES + 1): + try: + dr = f(r) / df(r) + except ZeroDivisionError: + log.error(f"Failed to find ratio due to zero division! Returning r={r0}") + return r0 + r = r - dr + # print(f"r({i})={r}") + # if the change in r is small, we are close enough + if abs(dr) < MIN_DR: + break + else: + log.error(f"Could not find ratio after {MAX_TRIES}! Returning r={r0}") + return r0 + return r + + +def backoff_delay(start, timeout=None, ratio=None, count: int = 0): + """generates `count` waits starting at `start` + sum of waits is `timeout` or each one is `ratio` bigger than the last + the last wait is always 0""" + # timeout or ratio must be set but not both + assert (timeout is None) ^ (ratio is None) + assert ratio is None or ratio > 0 + assert timeout is None or timeout >= start + assert (count > 1 or timeout is not None) and isinstance(count, int) + assert start > 0 + + if count == 0: + # Equation for auto-count is tuned to have a max of + # ~int(timeout) counts with a start wait of <0.01. + # Increasing start wait decreases count eg. + # backoff_delay(10, timeout=60) -> count = 5 + count = int( + (timeout / ((start + 0.05) ** (1 / 2)) + 2) // math.log(timeout + 2) + ) + + yield start + # if ratio is set: + # timeout = start * (1 - ratio**(count - 1)) / (1 - ratio) + if ratio is None: + ratio = find_ratio(start, count - 1, timeout) + + wait = start + # we have start and 0, so we only need to generate count - 2 + for _ in range(count - 2): + wait *= ratio + yield wait + yield 0 + return + + +ROOT_URL = "http://metadata.google.internal/computeMetadata/v1" + +class MetadataNotFoundError(Exception): + pass + +def get_metadata(path:str, silent=False) -> str: + """Get metadata relative to metadata/computeMetadata/v1""" + HEADERS = {"Metadata-Flavor": "Google"} + url = f"{ROOT_URL}/{path}" + try: + resp = requests_lib.get(url, headers=HEADERS) + resp.raise_for_status() + return resp.text + except requests_lib.exceptions.HTTPError: + if not silent: + log.warning(f"metadata not found ({url})") + raise MetadataNotFoundError(f"failed to get_metadata from {url}") + + +@lru_cache(maxsize=None) +def instance_metadata(path: str, silent:bool=False) -> str: + return get_metadata(f"instance/{path}", silent=silent) + +def instance_role(): + return instance_metadata("attributes/slurm_instance_role") + + +def instance_login_group(): + return instance_metadata("attributes/slurm_login_group") + + +def natural_sort(text): + def atoi(text): + return int(text) if text.isdigit() else text + + return [atoi(w) for w in re.split(r"(\d+)", text)] + + +def to_hostlist(names: Iterable[str]) -> str: + """ + Fast implementation of `hostlist` that doesn't invoke `scontrol` + IMPORTANT: + * Acts as `scontrol show hostlistsorted`, i.e. original order is not preserved + * Achieves worse compression than `scontrol show hostlist` for some cases + """ + pref = defaultdict(list) + tokenizer = re.compile(r"^(.*?)(\d*)$") + for name in filter(None, names): + matches = tokenizer.match(name) + assert matches, name + p, s = matches.groups() + pref[p].append(s) + + def _compress_suffixes(ss: List[str]) -> List[str]: + cur, res = None, [] + + def cur_repr(): + assert cur + nums, strs = cur + if nums[0] == nums[1]: + return strs[0] + return f"{strs[0]}-{strs[1]}" + + for s in sorted(ss, key=int): + n = int(s) + if cur is None: + cur = ((n, n), (s, s)) + continue + + nums, strs = cur + if n == nums[1] + 1: + cur = ((nums[0], n), (strs[0], s)) + else: + res.append(cur_repr()) + cur = ((n, n), (s, s)) + if cur: + res.append(cur_repr()) + return res + + res = [] + for p in sorted(pref.keys()): + sl = defaultdict(list) + for s in pref[p]: + sl[len(s)].append(s) + cs = [] + for ln in sorted(sl.keys()): + if ln == 0: + res.append(p) + else: + cs.extend(_compress_suffixes(sl[ln])) + if not cs: + continue + if len(cs) == 1 and "-" not in cs[0]: + res.append(f"{p}{cs[0]}") + else: + res.append(f"{p}[{','.join(cs)}]") + return ",".join(res) + +@lru_cache(maxsize=None) +def to_hostnames(nodelist: str) -> List[str]: + """make list of hostnames from hostlist expression""" + if not nodelist: + return [] # avoid degenerate invocation of scontrol + if isinstance(nodelist, str): + hostlist = nodelist + else: + hostlist = ",".join(nodelist) + hostnames = run(f"{lookup().scontrol} show hostnames {hostlist}").stdout.splitlines() + return hostnames + + +def retry_exception(exc) -> bool: + """return true for exceptions that should always be retried""" + msg = str(exc) + retry_errors = ( + "Rate Limit Exceeded", + "Quota Exceeded", + "Quota exceeded", + ) + return any(err in msg for err in retry_errors) + + +def ensure_execute(request): + """Handle rate limits and socket time outs""" + + for retry, wait in enumerate(backoff_delay(0.5, timeout=10 * 60, count=20)): + try: + return request.execute() + except googleapiclient.errors.HttpError as e: + if retry_exception(e): + log.error(f"retry:{retry} '{e}'") + sleep(wait) + continue + raise + + except socket.timeout as e: + # socket timed out, try again + log.debug(e) + + except Exception as e: + log.error(e, exc_info=True) + raise + + break + + +def batch_execute(requests, retry_cb=None, log_err=log.error): + """execute list or dict as batch requests + retry if retry_cb returns true + """ + BATCH_LIMIT = 1000 + if not isinstance(requests, dict): + requests = {str(k): v for k, v in enumerate(requests)} # rid generated here + done = {} + failed = {} + timestamps: List[float] = [] + rate_limited = False + + def batch_callback(rid, resp, exc): + nonlocal rate_limited + if exc is not None: + log_err(f"compute request exception {rid}: {exc}") + if retry_exception(exc): + rate_limited = True + else: + req = requests.pop(rid) + failed[rid] = (req, exc) + else: + # if retry_cb is set, don't move to done until it returns false + if retry_cb is None or not retry_cb(resp): + requests.pop(rid) + done[rid] = resp + + def batch_request(reqs): + batch = lookup().compute.new_batch_http_request(callback=batch_callback) + for rid, req in reqs: + batch.add(req, request_id=rid) + return batch + + while requests: + if timestamps: + timestamps = [stamp for stamp in timestamps if stamp > time()] + if rate_limited and timestamps: + stamp = next(iter(timestamps)) + sleep(max(stamp - time(), 0)) + rate_limited = False + # up to API_REQ_LIMIT (2000) requests + # in chunks of up to BATCH_LIMIT (1000) + batches = [ + batch_request(chunk) + for chunk in chunked(islice(requests.items(), API_REQ_LIMIT), BATCH_LIMIT) + ] + timestamps.append(time() + 100) + with ThreadPoolExecutor() as exe: + futures = [] + for batch in batches: + future = exe.submit(ensure_execute, batch) + futures.append(future) + for future in futures: + result = future.exception() + if result is not None: + raise result + + return done, failed + + +def get_operation_req(lkp: "Lookup", name: str, region: Optional[str]=None, zone: Optional[str]=None) -> Any: + if zone: + return lkp.compute.zoneOperations().get(project=lkp.project, zone=zone, operation=name) + elif region: + return lkp.compute.regionOperations().get(project=lkp.project, region=region, operation=name) + return lkp.compute.globalOperations().get(project=lkp.project, operation=name) + +def wait_request(operation, project: str): + """makes the appropriate wait request for a given operation""" + if "zone" in operation: + req = lookup().compute.zoneOperations().wait( + project=project, + zone=trim_self_link(operation["zone"]), + operation=operation["name"], + ) + elif "region" in operation: + req = lookup().compute.regionOperations().wait( + project=project, + region=trim_self_link(operation["region"]), + operation=operation["name"], + ) + else: + req = lookup().compute.globalOperations().wait( + project=project, operation=operation["name"] + ) + return req + + +def wait_for_operation(operation) -> Dict[str, Any]: + """wait for given operation""" + project = parse_self_link(operation["selfLink"]).project + wait_req = wait_request(operation, project=project) + + while True: + result = ensure_execute(wait_req) + if result["status"] == "DONE": + log_errors = " with errors" if "error" in result else "" + log.debug( + f"operation complete{log_errors}: type={result['operationType']}, name={result['name']}" + ) + return result + + + +def getThreadsPerCore(template) -> int: + if not template.machine_type.supports_smt: + return 1 + return template.advancedMachineFeatures.threadsPerCore or 2 + + +@retry( + max_retries=9, + init_wait_time=1, + warn_msg="Temporary failure in name resolution", + exc_type=socket.gaierror, +) +def host_lookup(host_name: str) -> str: + return socket.gethostbyname(host_name) + + +class Dumper(yaml.SafeDumper): + """Add representers for pathlib.Path and NSDict for yaml serialization""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.add_representer(NSDict, self.represent_nsdict) + self.add_multi_representer(Path, self.represent_path) + + @staticmethod + def represent_nsdict(dumper, data): + return dumper.represent_mapping("tag:yaml.org,2002:map", data.items()) + + @staticmethod + def represent_path(dumper, path): + return dumper.represent_scalar("tag:yaml.org,2002:str", str(path)) + + +@dataclass(frozen=True) +class ReservationDetails: + project: str + zone: str + name: str + policies: List[str] # names (not URLs) of resource policies + bulk_insert_name: str # name in format suitable for bulk insert (currently identical to user supplied name in long format) + deployment_type: Optional[str] + reservation_mode: Optional[str] + + @property + def dense(self) -> bool: + return self.deployment_type == "DENSE" + + @property + def calendar(self) -> bool: + return self.reservation_mode == "CALENDAR" + +@dataclass(frozen=True) +class FutureReservation: + project: str + zone: str + name: str + specific: bool + start_time: datetime + end_time: datetime + reservation_mode: Optional[str] + active_reservation: Optional[ReservationDetails] + + @property + def calendar(self) -> bool: + return self.reservation_mode == "CALENDAR" + +@dataclass +class Job: + id: int + name: Optional[str] = None + required_nodes: Optional[str] = None + job_state: Optional[str] = None + duration: Optional[timedelta] = None + +@dataclass(frozen=True) +class NodeState: + base: str + flags: frozenset + +class Lookup: + """Wrapper class for cached data access""" + + def __init__(self, cfg): + self._cfg = cfg + + @property + def cfg(self): + return self._cfg + + @property + def project(self): + return self.cfg.project or authentication_project() + + @cached_property + def control_addr(self) -> Optional[str]: + return self.cfg.get("slurm_control_addr", None) + + @property + def control_host(self): + return self.cfg.slurm_control_host + + @cached_property + def control_host_addr(self): + return self.control_addr or host_lookup(self.cfg.slurm_control_host) + + @property + def control_host_port(self): + return self.cfg.slurm_control_host_port + + @property + def endpoint_versions(self): + return self.cfg.endpoint_versions + + @property + def scontrol(self): + return Path(self.cfg.slurm_bin_dir or "") / "scontrol" + + @cached_property + def instance_role(self): + return instance_role() + + @cached_property + def instance_role_safe(self): + try: + role = self.instance_role + except Exception as e: + log.error(e) + role = None + return role + + @property + def is_controller(self): + return self.instance_role_safe == "controller" + + @property + def is_login_node(self): + return self.instance_role_safe == "login" + + @cached_property + def compute(self): + # TODO evaluate when we need to use google_app_cred_path + if self.cfg.google_app_cred_path: + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = self.cfg.google_app_cred_path + return compute_service() + + @cached_property + def hostname(self): + return socket.gethostname() + + @cached_property + def hostname_fqdn(self): + return socket.getfqdn() + + @cached_property + def zone(self): + return instance_metadata("zone") + + node_desc_regex = re.compile( + r"^(?P(?P[^\s\-]+)-(?P\S+))-(?P(?P\w+)|(?P\[[\d,-]+\]))$" + ) + + @lru_cache(maxsize=None) + def _node_desc(self, node_name): + """Get parts from node name""" + if not node_name: + node_name = self.hostname + # workaround below is for VMs whose hostname is FQDN + node_name_short = node_name.split(".")[0] + m = self.node_desc_regex.match(node_name_short) + if not m: + raise Exception(f"node name {node_name} is not valid") + return m.groupdict() + + def node_prefix(self, node_name=None): + return self._node_desc(node_name)["prefix"] + + def node_index(self, node: str) -> int: + """ node_index("cluster-nodeset-45") == 45 """ + suff = self._node_desc(node)["suffix"] + + if suff is None: + raise ValueError(f"Node {node} name does not end with numeric index") + return int(suff) + + def node_nodeset_name(self, node_name=None): + return self._node_desc(node_name)["nodeset"] + + def node_nodeset(self, node_name=None): + nodeset_name = self.node_nodeset_name(node_name) + if nodeset_name in self.cfg.nodeset_tpu: + return self.cfg.nodeset_tpu[nodeset_name] + + return self.cfg.nodeset[nodeset_name] + + def partition_is_tpu(self, part: str) -> bool: + """check if partition with name part contains a nodeset of type tpu""" + return len(self.cfg.partitions[part].partition_nodeset_tpu) > 0 + + + def node_is_tpu(self, node_name=None): + nodeset_name = self.node_nodeset_name(node_name) + return self.cfg.nodeset_tpu.get(nodeset_name) is not None + + def node_is_fr(self, node_name:str) -> bool: + return bool(self.node_nodeset(node_name).future_reservation) + + def is_dormant_fr_node(self, node_name:str) -> bool: + fr = self.future_reservation(self.node_nodeset(node_name)) + if not fr: + return False + return fr.active_reservation is None + + def node_is_dyn(self, node_name=None) -> bool: + nodeset = self.node_nodeset_name(node_name) + return self.cfg.nodeset_dyn.get(nodeset) is not None + + def node_template(self, node_name=None) -> str: + """ Self link of nodeset template """ + return self.node_nodeset(node_name).instance_template + + def node_template_info(self, node_name=None): + return self.template_info(self.node_template(node_name)) + + def node_region(self, node_name=None): + nodeset = self.node_nodeset(node_name) + return parse_self_link(nodeset.subnetwork).region + + def nodeset_prefix(self, nodeset_name): + return f"{self.cfg.slurm_cluster_name}-{nodeset_name}" + + def nodelist_range(self, nodeset_name: str, start: int, count: int) -> str: + assert 0 <= start and 0 < count + pref = self.nodeset_prefix(nodeset_name) + if count == 1: + return f"{pref}-{start}" + return f"{pref}-[{start}-{start + count - 1}]" + + def static_dynamic_sizes(self, nodeset: NSDict) -> Tuple[int, int]: + return (nodeset.node_count_static or 0, nodeset.node_count_dynamic_max or 0) + + def nodelist(self, nodeset) -> str: + cnt = sum(self.static_dynamic_sizes(nodeset)) + if cnt == 0: + return "" + return self.nodelist_range(nodeset.nodeset_name, 0, cnt) + + def nodenames(self, nodeset) -> Tuple[Iterable[str], Iterable[str]]: + pref = self.nodeset_prefix(nodeset.nodeset_name) + s_count, d_count = self.static_dynamic_sizes(nodeset) + return ( + (f"{pref}-{i}" for i in range(s_count)), + (f"{pref}-{i}" for i in range(s_count, s_count + d_count)), + ) + + def power_managed_nodesets(self) -> Iterable[NSDict]: + return chain(self.cfg.nodeset.values(), self.cfg.nodeset_tpu.values()) + + def is_power_managed_node(self, node_name: str) -> bool: + try: + ns = self.node_nodeset(node_name) + if ns is None: + return False + idx = int(self._node_desc(node_name)["suffix"]) + return idx < sum(self.static_dynamic_sizes(ns)) + except Exception: + return False + + def is_static_node(self, node_name: str) -> bool: + if not self.is_power_managed_node(node_name): + return False + idx = int(self._node_desc(node_name)["suffix"]) + return idx < self.node_nodeset(node_name).node_count_static + + @lru_cache(maxsize=None) + def slurm_nodes(self) -> Dict[str, NodeState]: + def parse_line(node_line) -> Tuple[str, NodeState]: + """turn node,state line to (node, NodeState)""" + # state flags include: CLOUD, COMPLETING, DRAIN, FAIL, POWERED_DOWN, + # POWERING_DOWN + node, fullstate = node_line.split(",") + state = fullstate.split("+") + state_tuple = NodeState(base=state[0], flags=frozenset(state[1:])) + return (node, state_tuple) + + cmd = ( + f"{self.scontrol} show nodes | " + r"grep -oP '^NodeName=\K(\S+)|\s+State=\K(\S+)' | " + r"paste -sd',\n'" + ) + node_lines = run(cmd, shell=True).stdout.rstrip().splitlines() + nodes = { + node: state + for node, state in map(parse_line, node_lines) + if "CLOUD" in state.flags or "DYNAMIC_NORM" in state.flags + } + return nodes + + def node_state(self, nodename: str) -> Optional[NodeState]: + state = self.slurm_nodes().get(nodename) + if state is not None: + return state + + # state is None => Slurm doesn't know this node, + # there are two reasons: + # * happy: + # * node belongs to removed nodeset + # * node belongs to downsized portion of nodeset + # * dynamic node that didn't register itself + # * unhappy: + # * there is a drift in Slurm and SlurmGCP configurations + # * `slurm_nodes` function failed to handle `scontrol show nodes`, + # TODO: make `slurm_nodes` robust by using `scontrol show nodes --json` + # In either of "unhappy" cases it's too dangerous to proceed - abort slurmsync. + try: + ns = self.node_nodeset(nodename) + except: + log.info(f"Unknown node {nodename}, belongs to unknown nodeset") + return None # Can't find nodeset, may be belongs to removed nodeset + + if self.node_is_dyn(nodename): + log.info(f"Unknown node {nodename}, belongs to dynamic nodeset") + return None # we can't make any judjment for dynamic nodes + + cnt = sum(self.static_dynamic_sizes(ns)) + if self.node_index(nodename) >= cnt: + log.info(f"Unknown node {nodename}, out of nodeset size boundaries ({cnt})") + return None # node belongs to downsized nodeset + + raise RuntimeError(f"Slurm does not recognize node {nodename}, potential misconfiguration.") + + + @lru_cache(maxsize=1) + def instances(self) -> Dict[str, Instance]: + instance_information_fields = [ + "creationTimestamp", + "name", + "resourceStatus", + "scheduling", + "status", + "labels.slurm_instance_role", + "zone", + "metadata", + ] + + instance_fields = ",".join(sorted(instance_information_fields)) + fields = f"items.zones.instances({instance_fields}),nextPageToken" + flt = f"labels.slurm_cluster_name={self.cfg.slurm_cluster_name} AND name:{self.cfg.slurm_cluster_name}-*" + act = self.compute.instances() + op = act.aggregatedList(project=self.project, fields=fields, filter=flt) + + instances = {} + while op is not None: + result = ensure_execute(op) + for zone in result.get("items", {}).values(): + for jo in zone.get("instances", []): + inst = Instance.from_json(jo) + if inst.name in instances: + log.error(f"Duplicate VM name {inst.name} across multiple zones") + instances[inst.name] = inst + op = act.aggregatedList_next(op, result) + return instances + + def instance(self, instance_name: str) -> Optional[Instance]: + return self.instances().get(instance_name) + + @lru_cache() + def _get_reservation(self, project: str, zone: str, name: str) -> Any: + """See https://cloud.google.com/compute/docs/reference/rest/v1/reservations""" + return self.compute.reservations().get( + project=project, zone=zone, reservation=name).execute() + + @lru_cache() + def get_mig(self, project: str, zone: str, self_link:str) -> Any: + """https://cloud.google.com/compute/docs/reference/rest/v1/instanceGroupManagers""" + return self.compute.instanceGroupManagers().get(project=project, zone=zone, instanceGroupManager=self_link).execute() + + @lru_cache + def get_mig_instances(self, project: str, zone: str, self_link:str) -> Any: + return self.compute.instanceGroupManagers().listManagedInstances(project=project, zone=zone, instanceGroupManager=self_link).execute() + + @lru_cache() + def get_mig_list(self, project: str, zone: str) -> Any: + """https://cloud.google.com/compute/docs/reference/rest/v1/instanceGroupManagers""" + return self.compute.instanceGroupManagers().list(project=project, zone=zone).execute() + + @lru_cache() + def _get_future_reservation(self, project:str, zone:str, name: str) -> Any: + """See https://cloud.google.com/compute/docs/reference/rest/v1/futureReservations""" + return self.compute.futureReservations().get(project=project, zone=zone, futureReservation=name).execute() + + def get_reservation_details(self, project:str, zone:str, name:str, bulk_insert_name:str) -> ReservationDetails: + reservation = self._get_reservation(project, zone, name) + + # Converts policy URLs to names, e.g.: + # projects/111111/regions/us-central1/resourcePolicies/zebra -> zebra + policies = [u.split("/")[-1] for u in reservation.get("resourcePolicies", {}).values()] + + return ReservationDetails( + project=project, + zone=zone, + name=name, + policies=policies, + deployment_type=reservation.get("deploymentType"), + reservation_mode=reservation.get("reservationMode"), + bulk_insert_name=bulk_insert_name) + + def nodeset_reservation(self, nodeset: NSDict) -> Optional[ReservationDetails]: + if not nodeset.reservation_name: + return None + + zones = list(nodeset.zone_policy_allow or []) + assert len(zones) == 1, "Only single zone is supported if using a reservation" + zone = zones[0] + + regex = re.compile(r'^projects/(?P[^/]+)/reservations/(?P[^/]+)(/.*)?$') + if not (match := regex.match(nodeset.reservation_name)): + raise ValueError( + f"Invalid reservation name: '{nodeset.reservation_name}', expected format is 'projects/PROJECT/reservations/NAME'" + ) + + project, name = match.group("project", "reservation") + return self.get_reservation_details(project, zone, name, nodeset.reservation_name) + + def future_reservation(self, nodeset: NSDict) -> Optional[FutureReservation]: + if not nodeset.future_reservation: + return None + + active_reservation = None + match = re.search(r'^projects/(?P[^/]+)/zones/(?P[^/]+)/futureReservations/(?P[^/]+)(/.*)?$', nodeset.future_reservation) + assert match, f"Invalid future reservation name '{nodeset.future_reservation}'" + project, zone, name = match.group("project","zone","name") + fr = self._get_future_reservation(project,zone,name) + + start_time = parse_gcp_timestamp(fr["timeWindow"]["startTime"]) + end_time = parse_gcp_timestamp(fr["timeWindow"]["endTime"]) + + if "autoCreatedReservations" in fr["status"] and (res:=fr["status"]["autoCreatedReservations"][0]): + if start_time <= now() <=end_time: + match = re.search(r'projects/(?P[^/]+)/zones/(?P[^/]+)/reservations/(?P[^/]+)(/.*)?$',res) + assert match, f"Unexpected reservation name '{res}'" + res_name = match.group("name") + bulk_insert_name = f"projects/{project}/reservations/{res_name}" + active_reservation = self.get_reservation_details(project, zone, res_name, bulk_insert_name) + + return FutureReservation( + project=project, + zone=zone, + name=name, + specific=fr["specificReservationRequired"], + start_time=start_time, + end_time=end_time, + reservation_mode=fr.get("reservationMode"), + active_reservation=active_reservation + ) + + @lru_cache(maxsize=1) + def machine_types(self): + field_names = "name,zone,guestCpus,memoryMb,accelerators" + fields = f"items.zones.machineTypes({field_names}),nextPageToken" + + machines: Dict[str, Dict[str, Any]] = defaultdict(dict) + act = self.compute.machineTypes() + op = act.aggregatedList(project=self.project, fields=fields) + while op is not None: + result = ensure_execute(op) + machine_iter = chain.from_iterable( + scope.get("machineTypes", []) for scope in result["items"].values() + ) + for machine in machine_iter: + name = machine["name"] + zone = machine["zone"] + machines[name][zone] = machine + + op = act.aggregatedList_next(op, result) + return machines + + def machine_type(self, name: str) -> MachineType: + custom_patt = re.compile( + r"((?P\w+)-)?custom-(?P\d+)-(?P\d+)" + ) + if match := custom_patt.match(name): + return MachineType( + name=name, + guest_cpus=int(match.group("cpus")), + memory_mb=int(match.group("mem")), + accelerators=[], + ) + + machines = self.machine_types() + if name not in machines: + raise Exception(f"machine type {name} not found") + per_zone = machines[name] + assert per_zone + return MachineType.from_json( + next(iter(per_zone.values())) # pick the first/any zone + ) + + def template_machine_conf(self, template_link): + template = self.template_info(template_link) + machine = template.machine_type + + machine_conf = NSDict() + machine_conf.boards = 1 # No information, assume 1 + machine_conf.sockets = machine.sockets + # the value below for SocketsPerBoard must be type int + machine_conf.sockets_per_board = machine_conf.sockets // machine_conf.boards + machine_conf.threads_per_core = 1 + _div = 2 if getThreadsPerCore(template) == 1 else 1 + machine_conf.cpus = ( + int(machine.guest_cpus / _div) if machine.supports_smt else machine.guest_cpus + ) + machine_conf.cores_per_socket = int(machine_conf.cpus / machine_conf.sockets) + # Because the actual memory on the host will be different than + # what is configured (e.g. kernel will take it). From + # experiments, about 16 MB per GB are used (plus about 400 MB + # buffer for the first couple of GB's. Using 30 MB to be safe. + gb = machine.memory_mb // 1024 + machine_conf.memory = machine.memory_mb - (400 + (30 * gb)) + return machine_conf + + @lru_cache(maxsize=None) + def template_info(self, template_link): + template_name = trim_self_link(template_link) + cache = file_cache.cache("template_cache") + + if cached := cache.get(template_name): + return NSDict(cached) + + template = ensure_execute( + self.compute.instanceTemplates().get( + project=self.project, instanceTemplate=template_name + ) + ).get("properties") + template = NSDict(template) + # name and link are not in properties, so stick them in + template.name = template_name + template.link = template_link + template.machine_type = self.machine_type(template.machineType) + # TODO delete metadata to reduce memory footprint? + # del template.metadata + + # translate gpus into an easier-to-read format + if template.machine_type.accelerators: + template.gpu = template.machine_type.accelerators[0] + elif template.guestAccelerators: + tga = template.guestAccelerators[0] + template.gpu = AcceleratorInfo( + type=tga.acceleratorType, + count=tga.acceleratorCount) + else: + template.gpu = None + + cache.set(template_name, template.to_dict()) + return template + + + def _parse_job_info(self, job_info: str) -> Job: + """Extract job details""" + if match:= re.search(r"JobId=(\d+)", job_info): + job_id = int(match.group(1)) + else: + raise ValueError(f"Job ID not found in the job info: {job_info}") + + if match:= re.search(r"TimeLimit=(?:(\d+)-)?(\d{2}):(\d{2}):(\d{2})", job_info): + days, hours, minutes, seconds = match.groups() + duration = timedelta( + days=int(days) if days else 0, + hours=int(hours), + minutes=int(minutes), + seconds=int(seconds) + ) + else: + duration = None + + if match := re.search(r"JobName=([^\n]+)", job_info): + name = match.group(1) + else: + name = None + + if match := re.search(r"JobState=(\w+)", job_info): + job_state = match.group(1) + else: + job_state = None + + if match := re.search(r"ReqNodeList=([^ ]+)", job_info): + required_nodes = match.group(1) + else: + required_nodes = None + + return Job(id=job_id, duration=duration, name=name, job_state=job_state, required_nodes=required_nodes) + + @lru_cache + def get_jobs(self) -> List[Job]: + res = run(f"{self.scontrol} show jobs", timeout=30) + + return [self._parse_job_info(job) for job in res.stdout.split("\n\n")[:-1]] + + @lru_cache + def job(self, job_id: int) -> Optional[Job]: + job_info = run(f"{self.scontrol} show jobid {job_id}", check=False).stdout.rstrip() + if not job_info: + return None + + return self._parse_job_info(job_info=job_info) + + @property + def etc_dir(self) -> Path: + return Path(self.cfg.output_dir or slurmdirs.etc) + + def controller_mount_server_ip(self) -> str: + return self.control_addr or self.control_host + + def normalize_ns_mount(self, ns: Union[dict, NSMount]) -> NSMount: + if isinstance(ns, NSMount): + return ns + + server_ip = ns.get("server_ip") or "$controller" + if server_ip == "$controller": + server_ip = self.controller_mount_server_ip() + + return NSMount( + server_ip=server_ip, + local_mount=Path(ns["local_mount"]), + remote_mount=Path(ns["remote_mount"]), + fs_type=ns["fs_type"], + mount_options=ns["mount_options"], + ) + + @property + def munge_mount(self) -> NSMount: + if self.cfg.munge_mount: + mnt = self.cfg.munge_mount + mnt.local_mount = mnt.local_mount or "/mnt/munge" + return self.normalize_ns_mount(mnt) + else: + return NSMount( + server_ip=self.controller_mount_server_ip(), + local_mount=Path("/mnt/munge"), + remote_mount=dirs.munge, + fs_type="nfs", + mount_options="defaults,hard,intr,_netdev", + ) + + @property + def slurm_key_mount(self) -> NSMount: + if self.cfg.slurm_key_mount: + mnt = self.cfg.slurm_key_mount + mnt.local_mount = mnt.local_mount or slurmdirs.key_distribution + return self.normalize_ns_mount(mnt) + else: + return NSMount( + server_ip=self.controller_mount_server_ip(), + local_mount=slurmdirs.key_distribution, + remote_mount=slurmdirs.key_distribution, + fs_type="nfs", + mount_options="defaults,hard,intr,_netdev", + ) + + def is_flex_node(self, node: str) -> bool: + try: + nodeset = self.node_nodeset(node) + if nodeset.dws_flex.use_bulk_insert: + return False #For legacy flex support + return bool(nodeset.dws_flex.enabled) + except: + return False + + def is_provisioning_flex_node(self, node:str) -> bool: + if not self.is_flex_node(node): + return False + if self.instance(node) is not None: + return True + + nodeset = self.node_nodeset(node) + zones = nodeset.zone_policy_allow + assert len(zones) == 1 + zone = zones[0] + + potential_migs=[] + mig_list=self.get_mig_list(self.project, zone) + + if not mig_list or not mig_list.get("items"): + return False + + for mig in mig_list["items"]: + if not mig.get("instanceTemplate"): #possibly an old MIG + return False + if mig["instanceTemplate"] == self.node_template(node) and mig["currentActions"]["creating"] > 0: + potential_migs.append(self.get_mig_instances(self.project, zone, trim_self_link(mig["selfLink"]))) + + if not potential_migs: + return False + + for instance_collection in potential_migs[0]["managedInstances"]: + if node in instance_collection["name"] and instance_collection["currentAction"]=="CREATING": + return True + return False + + def cluster_regions(self) -> list[str]: + """ + Returns all regions used in cluster + NOTE: only concerned with normal nodesets, + neither TPU, nor dynamic, nor login node, nor controller node are considered + """ + res = set() + for nodeset in self.cfg.nodeset.values(): + res.add(parse_self_link(nodeset.subnetwork).region) + return list(res) + + + +_lkp: Optional[Lookup] = None + +def _load_config() -> NSDict: + return NSDict(yaml.safe_load(CONFIG_FILE.read_text())) + +def lookup() -> Lookup: + global _lkp + if _lkp is None: + try: + cfg = _load_config() + except FileNotFoundError: + log.error(f"config file not found: {CONFIG_FILE}") + cfg = NSDict() # TODO: fail here, once all code paths are covered (mainly init_logging) + _lkp = Lookup(cfg) + return _lkp + +def update_config(cfg: NSDict) -> None: + global _lkp + _lkp = Lookup(cfg) + +def scontrol_reconfigure(lkp: Lookup) -> None: + log.info("Running scontrol reconfigure") + run(f"{lkp.scontrol} reconfigure") diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py new file mode 100644 index 0000000000..d1d77a1833 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py @@ -0,0 +1,124 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any + + +from dataclasses import dataclass, asdict +import util +import local_pubsub + +import logging +log = logging.getLogger() + +# Name of the topic +TOPIC = "watch_delete_vm_op" + +@dataclass(frozen=True) +class WatchDeleteVmOp_Message: + op_name: str + zone: str + node: str + +class WatchDeleteVmOp_Topic: + def __init__(self, topic: local_pubsub.Topic) -> None: + self._t = topic + + def publish(self, op: dict[str, Any], node: str) -> None: + assert op.get("operationType") == "delete" + assert op.get("zone") + assert node + + msg = WatchDeleteVmOp_Message(op_name=op["name"], zone=op["zone"], node=node) + self._t.publish(data=asdict(msg)) + + +def watch_delete_vm_op_topic() -> WatchDeleteVmOp_Topic: + return WatchDeleteVmOp_Topic(local_pubsub.topic(TOPIC)) + + +def _watch_op(lkp: util.Lookup, m: WatchDeleteVmOp_Message) -> bool: + """ + Processes VM delete-operation. + If operation is still running - do nothing + If operation failed - log error & remove op from watch list + If operation is done - remove op from watch list do nothing + + To avoid querying status for each op individually, use list of VM instances as + a source of data. Don't query op for instance X if instance X is not present + (presumably deleted). + NOTE: This optimization can lead to false-positives - + absence of error-logs in case op failed, but VM got deleted by other means. + + Returns True if message should be marked as processed (ack). + """ + + inst = lkp.instance(m.node) + + if not inst: + log.debug(f"Stop watching op {m.op_name}, VM {m.node} appears to be deleted") + return True # ack, potentially false-positive + + if inst.status == "TERMINATED": + log.debug(f"Stop watching op {m.op_name}, VM {m.node} is TERMINATED") + return True # ack, potentially false-positive + + if inst.status == "STOPPING": + log.debug(f"Skipping op {m.op_name}, VM {m.node} is STOPPING") + return False # try later + + try: + op = util.get_operation_req(lkp, m.op_name, zone=m.zone).execute() + except: + # TODO: consider less conservative handling, but be careful not to cause deadlettering. + log.exception(f"Failed to get operation {m.op_name}, will not retry") + return True # ack (remove) + + if op["status"] != "DONE": + log.debug(f"Watching op {m.op_name} is still not done ({op['status']})") + return False # try later + + if "error" in op: + log.error(f"Operation {m.op_name} to delete {m.node} finished with error: {op['error']}") + else: + log.debug(f"Operation {m.op_name} to delete {m.node} successfully finished") + return True # ack + + +def watch_vm_delete_ops(lkp: util.Lookup) -> None: + sub = local_pubsub.subscription(TOPIC) + + # Pull once instead of "pulling until empty", motivation: + # Bulk of cases processed by `_watch_op` relies on freshness of `lkp.instances`, + # `lkp.instances` are fetched once during run of `slurmsync`. + # Therefore we shouldn't try to re-process messages that has been already NACKed in this run, + # since they will be handled with the same `lkp.instance` as a previous attempt. + msgs = sub.pull(max_messages=1000) # 1000 is arbitrary number to be adjusted if needed. + log.debug(f"Processing {len(msgs)} delete VM operations") + # TODO: handle messages in butches to improve latency + for m in msgs: + try: + dm = WatchDeleteVmOp_Message(**m.data) + ack = _watch_op(lkp, dm) + except Exception: + log.exception(f"Failed to process the message {m.id}, removing") + ack = True + if ack: + sub.ack([m.id]) + else: + sub.modify_ack_deadline([m.id], deadline=0) # NACK + + + + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf new file mode 100644 index 0000000000..7163ec4c60 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf @@ -0,0 +1,500 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "bucket_name" { + description = <<-EOD + Name of GCS bucket to use. + EOD + type = string +} + +variable "bucket_dir" { + description = "Bucket directory for cluster files to be put into." + type = string + default = null +} + +variable "enable_debug_logging" { + type = bool + description = "Enables debug logging mode. Not for production use." + default = false +} + +variable "extra_logging_flags" { + type = map(bool) + description = "The only available flag is `trace_api`" + default = {} +} + +variable "project_id" { + description = "The GCP project ID." + type = string +} + +variable "enable_slurm_auth" { + description = < x... } + nodeset_map = { for k, vs in local.nodeset_map_ell : k => vs[0] } + + nodeset_tpu_map_ell = { for x in var.nodeset_tpu : x.nodeset_name => x... } + nodeset_tpu_map = { for k, vs in local.nodeset_tpu_map_ell : k => vs[0] } + + nodeset_dyn_map_ell = { for x in var.nodeset_dyn : x.nodeset_name => x... } + nodeset_dyn_map = { for k, vs in local.nodeset_dyn_map_ell : k => vs[0] } + + + no_reservation_affinity = { type : "NO_RESERVATION" } +} + +# NODESET +module "slurm_nodeset_template" { + source = "../../internal/slurm-gcp/instance_template" + for_each = local.nodeset_map + + project_id = var.project_id + slurm_cluster_name = local.slurm_cluster_name + slurm_instance_role = "compute" + slurm_bucket_path = module.slurm_files.slurm_bucket_path + + additional_disks = each.value.additional_disks + bandwidth_tier = each.value.bandwidth_tier + can_ip_forward = each.value.can_ip_forward + advanced_machine_features = each.value.advanced_machine_features + disk_auto_delete = each.value.disk_auto_delete + disk_labels = each.value.disk_labels + disk_resource_manager_tags = each.value.disk_resource_manager_tags + disk_size_gb = each.value.disk_size_gb + disk_type = each.value.disk_type + enable_confidential_vm = each.value.enable_confidential_vm + enable_oslogin = each.value.enable_oslogin + enable_shielded_vm = each.value.enable_shielded_vm + gpu = each.value.gpu + labels = merge(each.value.labels, { slurm_nodeset = each.value.nodeset_name }) + machine_type = each.value.machine_type + metadata = merge(each.value.metadata, local.universe_domain) + min_cpu_platform = each.value.min_cpu_platform + name_prefix = each.value.nodeset_name + on_host_maintenance = each.value.on_host_maintenance + preemptible = each.value.preemptible + region = each.value.region + resource_manager_tags = each.value.resource_manager_tags + spot = each.value.spot + termination_action = each.value.termination_action + service_account = each.value.service_account + shielded_instance_config = each.value.shielded_instance_config + source_image_family = each.value.source_image_family + source_image_project = each.value.source_image_project + source_image = each.value.source_image + subnetwork = each.value.subnetwork_self_link + additional_networks = each.value.additional_networks + access_config = each.value.access_config + tags = concat([local.slurm_cluster_name], each.value.tags) + + max_run_duration = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? each.value.dws_flex.max_run_duration : null + provisioning_model = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? "FLEX_START" : null + reservation_affinity = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? local.no_reservation_affinity : null +} + +module "nodeset_cleanup" { + source = "./modules/cleanup_compute" + for_each = local.nodeset_map + + nodeset = each.value + project_id = var.project_id + slurm_cluster_name = local.slurm_cluster_name + enable_cleanup_compute = var.enable_cleanup_compute + universe_domain = var.universe_domain + endpoint_versions = var.endpoint_versions + gcloud_path_override = var.gcloud_path_override + nodeset_template = module.slurm_nodeset_template[each.value.nodeset_name].self_link +} + +locals { + nodesets = [for name, ns in local.nodeset_map : { + nodeset_name = ns.nodeset_name + node_conf = ns.node_conf + dws_flex = ns.dws_flex + instance_template = module.slurm_nodeset_template[ns.nodeset_name].self_link + node_count_dynamic_max = ns.node_count_dynamic_max + node_count_static = ns.node_count_static + subnetwork = ns.subnetwork_self_link + reservation_name = ns.reservation_name + future_reservation = ns.future_reservation + maintenance_interval = ns.maintenance_interval + instance_properties_json = ns.instance_properties_json + enable_placement = ns.enable_placement + placement_max_distance = ns.placement_max_distance + network_storage = ns.network_storage + zone_target_shape = ns.zone_target_shape + zone_policy_allow = ns.zone_policy_allow + zone_policy_deny = ns.zone_policy_deny + enable_maintenance_reservation = ns.enable_maintenance_reservation + enable_opportunistic_maintenance = ns.enable_opportunistic_maintenance + }] +} + +# NODESET TPU +module "slurm_nodeset_tpu" { + source = "../../internal/slurm-gcp/nodeset_tpu" + for_each = local.nodeset_tpu_map + + project_id = var.project_id + node_count_dynamic_max = each.value.node_count_dynamic_max + node_count_static = each.value.node_count_static + nodeset_name = each.value.nodeset_name + zone = each.value.zone + node_type = each.value.node_type + accelerator_config = each.value.accelerator_config + tf_version = each.value.tf_version + preemptible = each.value.preemptible + preserve_tpu = each.value.preserve_tpu + enable_public_ip = each.value.enable_public_ip + service_account = each.value.service_account + data_disks = each.value.data_disks + docker_image = each.value.docker_image + subnetwork = each.value.subnetwork +} + +module "nodeset_cleanup_tpu" { + source = "./modules/cleanup_tpu" + for_each = local.nodeset_tpu_map + + nodeset = { + nodeset_name = each.value.nodeset_name + zone = each.value.zone + } + + project_id = var.project_id + slurm_cluster_name = local.slurm_cluster_name + enable_cleanup_compute = var.enable_cleanup_compute + universe_domain = var.universe_domain + endpoint_versions = var.endpoint_versions + gcloud_path_override = var.gcloud_path_override + + depends_on = [ + # Depend on controller network, as a best effort to avoid + # subnetwork resourceInUseByAnotherResource error + var.subnetwork_self_link + ] +} + +resource "google_storage_bucket_object" "parition_config" { + for_each = { for p in var.partitions : p.partition_name => p } + + bucket = module.slurm_files.bucket_name + name = "${module.slurm_files.bucket_dir}/partition_configs/${each.key}.yaml" + content = yamlencode(each.value) +} + +moved { + from = module.slurm_files.google_storage_bucket_object.parition_config + to = google_storage_bucket_object.parition_config +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf new file mode 100644 index 0000000000..32a7be1ad4 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf @@ -0,0 +1,191 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# BUCKET + +locals { + synt_suffix = substr(md5("${local.controller_project_id}${var.deployment_name}"), 0, 5) + synth_bucket_name = "${local.slurm_cluster_name}${local.synt_suffix}" + + bucket_name = var.create_bucket ? module.bucket[0].name : var.bucket_name +} + +module "bucket" { + source = "terraform-google-modules/cloud-storage/google" + version = "~> 6.1" + + count = var.create_bucket ? 1 : 0 + + location = var.region + names = [local.synth_bucket_name] + prefix = "slurm" + project_id = local.controller_project_id + + force_destroy = { + (local.synth_bucket_name) = true + } + + labels = merge(local.labels, { + slurm_cluster_name = local.slurm_cluster_name + }) +} + +# BUCKET IAMs +locals { + compute_sa = toset(flatten([for x in module.slurm_nodeset_template : x.service_account])) + compute_tpu_sa = toset(flatten([for x in module.slurm_nodeset_tpu : x.service_account])) + login_sa = toset(flatten([for x in module.login : x.service_account])) + + viewers = toset(flatten([ + "serviceAccount:${module.slurm_controller_template.service_account.email}", + formatlist("serviceAccount:%s", [for x in local.compute_sa : x.email]), + formatlist("serviceAccount:%s", [for x in local.compute_tpu_sa : x.email if x.email != null]), + formatlist("serviceAccount:%s", [for x in local.login_sa : x.email]), + ])) +} + + +resource "google_storage_bucket_iam_member" "viewers" { + for_each = local.viewers + bucket = local.bucket_name + role = "roles/storage.objectViewer" + member = each.value +} + +resource "google_storage_bucket_iam_member" "legacy_readers" { + for_each = local.viewers + bucket = local.bucket_name + role = "roles/storage.legacyBucketReader" + member = each.value +} + +locals { + daos_ns = [ + for ns in var.network_storage : + ns if ns.fs_type == "daos" + ] + + daos_client_install_runners = [ + for ns in local.daos_ns : + ns.client_install_runner if ns.client_install_runner != null + ] + + daos_mount_runners = [ + for ns in local.daos_ns : + ns.mount_runner if ns.mount_runner != null + ] + + daos_network_storage_runners = concat( + local.daos_client_install_runners, + local.daos_mount_runners, + ) + + daos_install_mount_script = { + filename = "ghpc_daos_mount.sh" + content = length(local.daos_ns) > 0 ? module.daos_network_storage_scripts[0].startup_script : "" + } + + common_scripts = length(local.daos_ns) > 0 ? [local.daos_install_mount_script] : [] +} + +# SLURM FILES +locals { + ghpc_startup_script_controller = concat( + local.common_scripts, + [{ + filename = "ghpc_startup.sh" + content = var.controller_startup_script + }]) + + controller_state_disk = { + device_name : try(google_compute_disk.controller_disk[0].name, null) + } + + + nodeset_startup_scripts = { for k, v in local.nodeset_map : k => concat(local.common_scripts, v.startup_script) } +} + +module "daos_network_storage_scripts" { + count = length(local.daos_ns) > 0 ? 1 : 0 + + source = "../../../../modules/scripts/startup-script" + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.daos_network_storage_runners +} + +module "slurm_files" { + source = "./modules/slurm_files" + + project_id = var.project_id + slurm_cluster_name = local.slurm_cluster_name + bucket_dir = var.bucket_dir + bucket_name = local.bucket_name + controller_network_attachment = var.controller_network_attachment + + slurmdbd_conf_tpl = var.slurmdbd_conf_tpl + slurm_conf_tpl = var.slurm_conf_tpl + slurm_conf_template = var.slurm_conf_template + cgroup_conf_tpl = var.cgroup_conf_tpl + cloud_parameters = var.cloud_parameters + cloudsql_secret = try( + one(google_secret_manager_secret_version.cloudsql_version[*].id), + null) + + controller_startup_scripts = local.ghpc_startup_script_controller + controller_startup_scripts_timeout = var.controller_startup_scripts_timeout + nodeset_startup_scripts = local.nodeset_startup_scripts + compute_startup_scripts_timeout = var.compute_startup_scripts_timeout + controller_state_disk = local.controller_state_disk + + enable_debug_logging = var.enable_debug_logging + extra_logging_flags = var.extra_logging_flags + + enable_slurm_auth = var.enable_slurm_auth + + enable_bigquery_load = var.enable_bigquery_load + enable_external_prolog_epilog = var.enable_external_prolog_epilog + enable_chs_gpu_health_check_prolog = var.enable_chs_gpu_health_check_prolog + enable_chs_gpu_health_check_epilog = var.enable_chs_gpu_health_check_epilog + epilog_scripts = var.epilog_scripts + prolog_scripts = var.prolog_scripts + task_epilog_scripts = var.task_epilog_scripts + task_prolog_scripts = var.task_prolog_scripts + + disable_default_mounts = !var.enable_default_mounts + network_storage = [ + for storage in var.network_storage : { + server_ip = storage.server_ip, + remote_mount = storage.remote_mount, + local_mount = storage.local_mount, + fs_type = storage.fs_type, + mount_options = storage.mount_options + } + if storage.fs_type != "daos" + ] + + nodeset = local.nodesets + nodeset_dyn = values(local.nodeset_dyn_map) + # Use legacy format for now + nodeset_tpu = values(module.slurm_nodeset_tpu)[*] + + + depends_on = [module.bucket] + + # Providers + endpoint_versions = var.endpoint_versions +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf new file mode 100644 index 0000000000..0641cea7a8 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf @@ -0,0 +1,77 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # Currently supported images and projects + known_project_families = { + schedmd-slurm-public = [ + "slurm-gcp-6-9-debian-11", + "slurm-gcp-6-9-hpc-rocky-linux-8", + "slurm-gcp-6-9-ubuntu-2004-lts", + "slurm-gcp-6-9-ubuntu-2204-lts-arm64" + ] + } + + # This approach to "hacking" the project name allows a chain of Terraform + # calls to set the instance source_image (boot disk) with a "relative + # resource name" that passes muster with VPC Service Control rules + # + # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 + # https://cloud.google.com/apis/design/resource_names#relative_resource_name + source_image_project_normalized = (can(var.instance_image.family) ? + "projects/${data.google_compute_image.slurm.project}/global/images/family" : + "projects/${data.google_compute_image.slurm.project}/global/images" + ) + source_image_family = can(var.instance_image.family) ? data.google_compute_image.slurm.family : "" + source_image = can(var.instance_image.name) ? data.google_compute_image.slurm.name : "" +} + +data "google_compute_image" "slurm" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = var.instance_image.project + + lifecycle { + precondition { + condition = length(regexall("^projects/.+?/global/images/family$", var.instance_image.project)) == 0 + error_message = "The \"project\" field in var.instance_image no longer supports a long-form ending in \"family\". Specify only the project ID." + } + + postcondition { + condition = var.instance_image_custom || contains(keys(local.known_project_families), self.project) + error_message = <<-EOD + Images in project ${self.project} are not published by SchedMD. Images must be created by compatible releases of the Terraform and Packer modules following the guidance at https://goo.gle/hpc-slurm-images. Set var.instance_image_custom to true to silence this error and acknowledge that you are using a compatible image. + EOD + } + postcondition { + condition = !contains(keys(local.known_project_families), self.project) || try(contains(local.known_project_families[self.project], self.family), false) + error_message = <<-EOD + Image family ${self.family} published by SchedMD in project ${self.project} is not compatible with this release of the Terraform Slurm modules. Select from known compatible releases: + ${join("\n", [for p in try(local.known_project_families[self.project], []) : "\t\"${p}\""])} + EOD + } + postcondition { + condition = var.disk_size_gb >= self.disk_size_gb + error_message = "'disk_size_gb: ${var.disk_size_gb}' is smaller than the image size (${self.disk_size_gb}GB), please increase the blueprint disk size" + } + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf new file mode 100644 index 0000000000..cd7a47f0a3 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf @@ -0,0 +1,809 @@ +/** + * Copyright (C) SchedMD LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +########### +# GENERAL # +########### + +variable "project_id" { + type = string + description = "Project ID to create resources in." +} + +variable "deployment_name" { + description = "Name of the deployment." + type = string +} + +variable "slurm_cluster_name" { + type = string + description = <<-EOD + Cluster name, used for resource naming and slurm accounting. + If not provided it will default to the first 8 characters of the deployment name (removing any invalid characters). + EOD + default = null + + validation { + condition = var.slurm_cluster_name == null || can(regex("^[a-z](?:[a-z0-9]{0,9})$", var.slurm_cluster_name)) + error_message = "Variable 'slurm_cluster_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,9})$'." + } +} + +variable "region" { + type = string + description = "The default region to place resources in." +} + +variable "zone" { + type = string + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_compute_image.slurm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    disk_name                  = string
    device_name                = string
    disk_type                  = string
    disk_size_gb               = number
    disk_labels                = map(string)
    auto_delete                = bool
    boot                       = bool
    disk_resource_manager_tags = map(string)
  }))
| `[]` | no | +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    network            = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    queue_count        = optional(number)
    stack_type         = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
  }))
| `[]` | no | +| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | +| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | +| [disable\_login\_public\_ips](#input\_disable\_login\_public\_ips) | DEPRECATED: Use `enable_login_public_ips` instead. | `bool` | `null` | no | +| [disable\_smt](#input\_disable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | +| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | +| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `50` | no | +| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-ssd"` | no | +| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_login\_public\_ips](#input\_enable\_login\_public\_ips) | If set to true. The login node will have a random public IP assigned to it. | `bool` | `false` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | +| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm controller VM instance.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-9-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | +| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | +| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for login nodes. | `string` | `null` | no | +| [labels](#input\_labels) | Labels, provided as a map. | `map(string)` | `{}` | no | +| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"c2-standard-4"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | +| [name\_prefix](#input\_name\_prefix) | Unique name prefix for login nodes. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all login groups. | `string` | n/a | yes | +| [num\_instances](#input\_num\_instances) | Number of instances to create. This value is ignored if static\_ips is provided. | `number` | `1` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy. | `string` | `"MIGRATE"` | no | +| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | +| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | +| [region](#input\_region) | Region where the instances should be created. | `string` | `null` | no | +| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the login instances. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the login instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances. | `list(string)` | `[]` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | +| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | +| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be
spread across available zones in the region. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [login\_nodes](#output\_login\_nodes) | Slurm login instance definition. | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf new file mode 100644 index 0000000000..c12ee4627b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf @@ -0,0 +1,108 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-login", ghpc_role = "scheduler" }) +} + +module "gpu" { + source = "../../../../modules/internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + metadata = merge( + local.disable_automatic_updates_metadata, + var.metadata + ) + + additional_disks = [ + for ad in var.additional_disks : { + disk_name = ad.disk_name + device_name = ad.device_name + disk_type = ad.disk_type + disk_size_gb = ad.disk_size_gb + disk_labels = merge(ad.disk_labels, local.labels) + auto_delete = ad.auto_delete + boot = ad.boot + disk_resource_manager_tags = ad.var.disk_resource_manager_tags + } + ] + + public_access_config = [{ nat_ip = null, network_tier = null }] + + service_account = { + email = var.service_account_email + scopes = var.service_account_scopes + } + + # lower, replace `_` with `-`, and remove any non-alphanumeric characters + group_name = replace( + replace( + lower(var.name_prefix), + "_", "-"), + "/[^-a-z0-9]/", "") + + + login_node = { + group_name = local.group_name + disk_auto_delete = var.disk_auto_delete + disk_labels = merge(var.disk_labels, local.labels) + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + disk_resource_manager_tags = var.disk_resource_manager_tags + additional_disks = local.additional_disks + additional_networks = var.additional_networks + + can_ip_forward = var.can_ip_forward + advanced_machine_features = var.advanced_machine_features + + enable_confidential_vm = var.enable_confidential_vm + access_config = var.enable_login_public_ips ? local.public_access_config : [] + enable_oslogin = var.enable_oslogin + enable_shielded_vm = var.enable_shielded_vm + shielded_instance_config = var.shielded_instance_config + + gpu = one(local.guest_accelerator) + labels = local.labels + machine_type = var.machine_type + metadata = local.metadata + min_cpu_platform = var.min_cpu_platform + num_instances = var.num_instances + on_host_maintenance = var.on_host_maintenance + preemptible = var.preemptible + region = var.region + resource_manager_tags = var.resource_manager_tags + zone = var.zone + + service_account = local.service_account + + source_image_family = local.source_image_family # requires source_image_logic.tf + source_image_project = local.source_image_project_normalized # requires source_image_logic.tf + source_image = local.source_image # requires source_image_logic.tf + + static_ips = var.static_ips + bandwidth_tier = var.bandwidth_tier + + subnetwork = var.subnetwork_self_link + tags = var.tags + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml new file mode 100644 index 0000000000..47f003258e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] +ghpc: + inject_module_id: name_prefix + has_to_be_used: true diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf new file mode 100644 index 0000000000..e700542794 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf @@ -0,0 +1,18 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "login_nodes" { + description = "Slurm login instance definition." + value = [local.login_node] +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf new file mode 100644 index 0000000000..0641cea7a8 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf @@ -0,0 +1,77 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # Currently supported images and projects + known_project_families = { + schedmd-slurm-public = [ + "slurm-gcp-6-9-debian-11", + "slurm-gcp-6-9-hpc-rocky-linux-8", + "slurm-gcp-6-9-ubuntu-2004-lts", + "slurm-gcp-6-9-ubuntu-2204-lts-arm64" + ] + } + + # This approach to "hacking" the project name allows a chain of Terraform + # calls to set the instance source_image (boot disk) with a "relative + # resource name" that passes muster with VPC Service Control rules + # + # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 + # https://cloud.google.com/apis/design/resource_names#relative_resource_name + source_image_project_normalized = (can(var.instance_image.family) ? + "projects/${data.google_compute_image.slurm.project}/global/images/family" : + "projects/${data.google_compute_image.slurm.project}/global/images" + ) + source_image_family = can(var.instance_image.family) ? data.google_compute_image.slurm.family : "" + source_image = can(var.instance_image.name) ? data.google_compute_image.slurm.name : "" +} + +data "google_compute_image" "slurm" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = var.instance_image.project + + lifecycle { + precondition { + condition = length(regexall("^projects/.+?/global/images/family$", var.instance_image.project)) == 0 + error_message = "The \"project\" field in var.instance_image no longer supports a long-form ending in \"family\". Specify only the project ID." + } + + postcondition { + condition = var.instance_image_custom || contains(keys(local.known_project_families), self.project) + error_message = <<-EOD + Images in project ${self.project} are not published by SchedMD. Images must be created by compatible releases of the Terraform and Packer modules following the guidance at https://goo.gle/hpc-slurm-images. Set var.instance_image_custom to true to silence this error and acknowledge that you are using a compatible image. + EOD + } + postcondition { + condition = !contains(keys(local.known_project_families), self.project) || try(contains(local.known_project_families[self.project], self.family), false) + error_message = <<-EOD + Image family ${self.family} published by SchedMD in project ${self.project} is not compatible with this release of the Terraform Slurm modules. Select from known compatible releases: + ${join("\n", [for p in try(local.known_project_families[self.project], []) : "\t\"${p}\""])} + EOD + } + postcondition { + condition = var.disk_size_gb >= self.disk_size_gb + error_message = "'disk_size_gb: ${var.disk_size_gb}' is smaller than the image size (${self.disk_size_gb}GB), please increase the blueprint disk size" + } + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf new file mode 100644 index 0000000000..8b51cf1ed2 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf @@ -0,0 +1,419 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +variable "project_id" { # tflint-ignore: terraform_unused_declarations + type = string + description = "Project ID to create resources in." +} + +variable "region" { + type = string + description = "Region where the instances should be created." + default = null +} + +variable "zone" { + type = string + description = <<-EOD + Zone where the instances should be created. If not specified, instances will be + spread across available zones in the region. + EOD + default = null +} + +variable "name_prefix" { + type = string + description = <<-EOD + Unique name prefix for login nodes. Automatically populated by the module id if not set. + If setting manually, ensure a unique value across all login groups. + EOD +} + +variable "num_instances" { + type = number + description = "Number of instances to create. This value is ignored if static_ips is provided." + default = 1 +} + +variable "resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} +} + +variable "disk_type" { + type = string + description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." + default = "pd-ssd" +} + +variable "disk_size_gb" { + type = number + description = "Boot disk size in GB." + default = 50 +} + +variable "disk_auto_delete" { + type = bool + description = "Whether or not the boot disk should be auto-deleted." + default = true +} + +variable "disk_labels" { + description = "Labels specific to the boot disk. These will be merged with var.labels." + type = map(string) + default = {} +} + +variable "disk_resource_manager_tags" { + description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." + type = map(string) + default = {} + validation { + condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) + error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" + } + validation { + condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) + error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" + } +} + +variable "additional_disks" { + type = list(object({ + disk_name = string + device_name = string + disk_type = string + disk_size_gb = number + disk_labels = map(string) + auto_delete = bool + boot = bool + disk_resource_manager_tags = map(string) + })) + description = "List of maps of disks." + default = [] +} + +variable "additional_networks" { + description = "Additional network interface details for GCE, if any." + default = [] + type = list(object({ + access_config = optional(list(object({ + nat_ip = string + network_tier = string + })), []) + alias_ip_range = optional(list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })), []) + ipv6_access_config = optional(list(object({ + network_tier = string + })), []) + network = optional(string) + network_ip = optional(string, "") + nic_type = optional(string) + queue_count = optional(number) + stack_type = optional(string) + subnetwork = optional(string) + subnetwork_project = optional(string) + })) + nullable = false +} + +variable "advanced_machine_features" { + description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" + type = object({ + enable_nested_virtualization = optional(bool) + threads_per_core = optional(number) + turbo_mode = optional(string) + visible_core_count = optional(number) + performance_monitoring_unit = optional(string) + enable_uefi_networking = optional(bool) + }) + default = { + threads_per_core = 1 # disable SMT by default + } +} + +variable "enable_smt" { # tflint-ignore: terraform_unused_declarations + type = bool + description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + default = null + validation { + condition = var.enable_smt == null + error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + } +} + +variable "disable_smt" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + type = bool + default = null + validation { + condition = var.disable_smt == null + error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." + } +} + +variable "static_ips" { + type = list(string) + description = "List of static IPs for VM instances." + default = [] +} + +variable "bandwidth_tier" { + description = < +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.0 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [condor\_version](#input\_condor\_version) | Yum/DNF-compatible version string; leave unset to use latest 23.0 LTS release (examples: "23.0.0","23.*")) | `string` | `"23.*"` | no | +| [enable\_docker](#input\_enable\_docker) | Install and enable docker daemon alongside HTCondor | `bool` | `true` | no | +| [http\_proxy](#input\_http\_proxy) | Set system default web (http and https) proxy for Windows HTCondor installation | `string` | `""` | no | +| [python\_windows\_installer\_url](#input\_python\_windows\_installer\_url) | URL of Python installer for Windows | `string` | `"https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [gcp\_service\_list](#output\_gcp\_service\_list) | Google Cloud APIs required by HTCondor | +| [runners](#output\_runners) | Runner to install HTCondor using startup-scripts | +| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | Windows PowerShell script to install HTCondor | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py new file mode 100644 index 0000000000..77bafa0310 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py @@ -0,0 +1,417 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script for resizing managed instance group (MIG) cluster size based +# on the number of jobs in the Condor Queue. + +from absl import app +from absl import flags +from collections import OrderedDict +from datetime import datetime +from pprint import pprint +from googleapiclient import discovery +from oauth2client.client import GoogleCredentials + +import argparse +import os +import math +import time +import htcondor +import classad + +parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument("--p", required=True, help="Project id", type=str) +parser.add_argument( + "--z", + required=True, + help="Name of GCP zone where the managed instance group is located", + type=str, +) +parser.add_argument( + "--r", + required=True, + help="Name of GCP region where the managed instance group is located", + type=str, +) +parser.add_argument( + "--mz", + required=False, + help="Enabled multizone (regional) managed instance group", + action="store_true", +) +parser.add_argument( + "--g", required=True, help="Name of the managed instance group", type=str +) +parser.add_argument( + "--i", + default=0, + help="Minimum number of idle compute instances", + type=int +) +parser.add_argument( + "--c", required=True, help="Maximum number of compute instances", type=int +) +parser.add_argument( + "--v", + default=0, + help="Increase output verbosity. 1-show basic debug info. 2-show detail debug info", + type=int, + choices=[0, 1, 2], +) +parser.add_argument( + "--d", + default=0, + help="Dry Run, default=0, if 1, then no scaling actions", + type=int, + choices=[0, 1], +) + +args = parser.parse_args() + +class AutoScaler: + def __init__(self, multizone=False): + + self.multizone = multizone + # Obtain credentials + self.credentials = GoogleCredentials.get_application_default() + self.service = discovery.build("compute", "v1", credentials=self.credentials) + + if self.multizone: + self.instanceGroupManagers = self.service.regionInstanceGroupManagers() + else: + self.instanceGroupManagers = self.service.instanceGroupManagers() + + # Remove specified instances from MIG and decrease MIG size + def deleteFromMig(self, node_self_links): + requestDelInstance = self.instanceGroupManagers.deleteInstances( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + body={ "instances": node_self_links }, + ) + + # execute if not a dry-run + if not self.dryrun: + response = requestDelInstance.execute() + if self.debug > 0: + pprint(response) + return response + return "Dry Run" + + def getInstanceTemplateInfo(self): + requestTemplateName = self.instanceGroupManagers.get( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + fields="instanceTemplate", + ) + responseTemplateName = requestTemplateName.execute() + template_name = "" + + if self.debug > 1: + print("Request for the template name") + pprint(responseTemplateName) + + if len(responseTemplateName) > 0: + template_url = responseTemplateName.get("instanceTemplate") + template_url_partitioned = template_url.split("/") + template_name = template_url_partitioned[len(template_url_partitioned) - 1] + + requestInstanceTemplate = self.service.instanceTemplates().get( + project=self.project, instanceTemplate=template_name, fields="properties" + ) + responseInstanceTemplateInfo = requestInstanceTemplate.execute() + + if self.debug > 1: + print("Template information") + pprint(responseInstanceTemplateInfo["properties"]) + + machine_type = responseInstanceTemplateInfo["properties"]["machineType"] + is_spot = responseInstanceTemplateInfo["properties"]["scheduling"][ + "preemptible" + ] + if self.debug > 0: + print("Machine Type: " + machine_type) + print("Is spot: " + str(is_spot)) + request = self.service.machineTypes().get( + project=self.project, zone=self.zone, machineType=machine_type + ) + response = request.execute() + guest_cpus = response["guestCpus"] + if self.debug > 1: + print("Machine information") + pprint(responseInstanceTemplateInfo["properties"]) + if self.debug > 0: + print("Guest CPUs: " + str(guest_cpus)) + + instanceTemplateInfo = { + "machine_type": machine_type, + "is_spot": is_spot, + "guest_cpus": guest_cpus, + } + return instanceTemplateInfo + + def scale(self): + # diagnosis + if self.debug > 1: + print("Launching autoscaler.py with the following arguments:") + print("project_id: " + self.project) + print("zone: " + self.zone) + print("region: " + self.region) + print(f"multizone: {self.multizone}") + print("group_manager: " + self.instance_group_manager) + print("computeinstancelimit: " + str(self.compute_instance_limit)) + print("debuglevel: " + str(self.debug)) + + if self.multizone: + self.zoneargs = {"region": self.region} + else: + self.zoneargs = {"zone": self.zone} + + # Each HTCondor scheduler (SchedD), maintains a list of jobs under its + # stewardship. A full list of Job ClassAd attributes can be found at + # https://htcondor.readthedocs.io/en/latest/classad-attributes/job-classad-attributes.html + schedd = htcondor.Schedd() + # encourage the job queue to start a new negotiation cycle; there are + # internal unconfigurable rate limits so not guaranteed; this is not + # strictly required for success, but may reduce latency of autoscaling + schedd.reschedule() + REQUEST_CPUS_ATTRIBUTE = "RequestCpus" + REQUEST_GPUS_ATTRIBUTE = "RequestGpus" + REQUEST_MEMORY_ATTRIBUTE = "RequestMemory" + job_attributes = [ + REQUEST_CPUS_ATTRIBUTE, + REQUEST_GPUS_ATTRIBUTE, + REQUEST_MEMORY_ATTRIBUTE, + ] + + instanceTemplateInfo = self.getInstanceTemplateInfo() + self.is_spot = instanceTemplateInfo["is_spot"] + self.cores_per_node = instanceTemplateInfo["guest_cpus"] + print(f"MIG is configured for Spot pricing: {self.is_spot}") + print("Number of CPU per compute node: " + str(self.cores_per_node)) + + # this query will constrain the search for jobs to those that either + # require spot VMs or do not require Spot VMs based on whether the + # VM instance template is configured for Spot pricing + spot_query = classad.ExprTree(f"RequireId == \"{self.instance_group_manager}\"") + + # For purpose of scaling a Managed Instance Group, count only jobs that + # are idle and likely participated in a negotiation cycle (there does + # not appear to be a single classad attribute for this). + # https://htcondor.readthedocs.io/en/latest/classad-attributes/job-classad-attributes.html#JobStatus + LAST_CYCLE_ATTRIBUTE = "LastNegotiationCycleTime0" + coll = htcondor.Collector() + negotiator_ad = coll.query(htcondor.AdTypes.Negotiator, projection=[LAST_CYCLE_ATTRIBUTE]) + if len(negotiator_ad) != 1: + print(f"There should be exactly 1 negotiator in the pool. There is {len(negotiator_ad)}") + exit() + last_negotiation_cycle_time = negotiator_ad[0].get(LAST_CYCLE_ATTRIBUTE) + if not last_negotiation_cycle_time: + print(f"The negotiator has not yet started a match cycle. Exiting auto-scaling.") + exit() + + print(f"Last negotiation cycle occurred at: {datetime.fromtimestamp(last_negotiation_cycle_time)}") + idle_job_query = classad.ExprTree(f"JobStatus == 1 && QDate < {last_negotiation_cycle_time}") + idle_job_ads = schedd.query(constraint=idle_job_query.and_(spot_query), + projection=job_attributes) + + total_idle_request_cpus = sum(j[REQUEST_CPUS_ATTRIBUTE] for j in idle_job_ads) + print(f"Total CPUs requested by idle jobs: {total_idle_request_cpus}") + + if self.debug > 1: + print("Information about the compute instance template") + pprint(instanceTemplateInfo) + + # Calculate the minimum number of instances that, for fully packed + # execute points, could satisfy current job queue + min_hosts_for_idle_jobs = math.ceil(total_idle_request_cpus / self.cores_per_node) + if self.debug > 0: + print(f"Minimum hosts needed: {total_idle_request_cpus} / {self.cores_per_node} = {min_hosts_for_idle_jobs}") + + # Get current number of instances in the MIG + requestGroupInfo = self.instanceGroupManagers.get( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + ) + responseGroupInfo = requestGroupInfo.execute() + current_target = responseGroupInfo["targetSize"] + print(f"Current MIG target size: {current_target}") + + # Find instances that are being modified by the MIG (currentAction is + # any value other than "NONE"). A common reason an instance is modified + # is it because it has failed a health check. + reqModifyingInstances = self.instanceGroupManagers.listManagedInstances( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + filter="currentAction != \"NONE\"", + orderBy="creationTimestamp desc" + ) + respModifyingInstances = reqModifyingInstances.execute() + + # Find VMs that are idle (no dynamic slots created from partitionable + # slots) in the MIG handled by this autoscaler + filter_idle_vms = classad.ExprTree(f"PartitionableSlot && NumDynamicSlots==0") + filter_claimed_vms = classad.ExprTree(f"PartitionableSlot && NumDynamicSlots>0") + filter_mig = classad.ExprTree(f"regexp(\".*/{self.instance_group_manager}$\", CloudCreatedBy)") + # A full list of Machine (StartD) ClassAd attributes can be found at + # https://htcondor.readthedocs.io/en/latest/classad-attributes/machine-classad-attributes.html + idle_node_ads = coll.query(htcondor.AdTypes.Startd, + constraint=filter_idle_vms.and_(filter_mig), + projection=["Machine", "CloudZone"]) + + NODENAME_ATTRIBUTE = "Machine" + claimed_node_ads = coll.query(htcondor.AdTypes.Startd, + constraint=filter_claimed_vms.and_(filter_mig), + projection=[NODENAME_ATTRIBUTE]) + claimed_nodes = [ ad[NODENAME_ATTRIBUTE].split(".")[0] for ad in claimed_node_ads] + + # treat OrderedDict as a set by ignoring key values; this set will + # contain VMs we would consider deleting, in inverse order of + # their readiness to join pool (creating, unhealthy, healthy+idle) + idle_nodes = OrderedDict() + try: + modifyingInstances = respModifyingInstances["managedInstances"] + except KeyError: + modifyingInstances = [] + + print(f"There are {len(modifyingInstances)} VMs being modified by the managed instance group") + + # there is potential for nodes in MIG health check "VERIFYING" state + # to have already joined the pool and be running jobs + for instance in modifyingInstances: + self_link = instance["instance"] + node_name = self_link.rsplit("/", 1)[-1] + if node_name not in claimed_nodes: + idle_nodes[self_link] = "modifying" + + for ad in idle_node_ads: + node = ad["Machine"].split(".")[0] + zone = ad["CloudZone"] + self_link = "https://www.googleapis.com/compute/v1/projects/" + \ + self.project + "/zones/" + zone + "/instances/" + node + # there is potential for nodes in MIG health check "VERIFYING" state + # to have already joined the pool and be idle; delete them last + if self_link in idle_nodes: + idle_nodes.move_to_end(self_link) + idle_nodes[self_link] = "idle" + n_idle = len(idle_nodes) + + print(f"There are {n_idle} VMs being modified or idle in the pool") + if self.debug > 1: + print("Listing idle nodes:") + pprint(idle_nodes) + + # always keep size tending toward the minimum idle VMs requested + new_target = current_target + self.compute_instance_min_idle - n_idle + min_hosts_for_idle_jobs + if new_target > self.compute_instance_limit: + self.size = self.compute_instance_limit + print(f"MIG target size will be limited by {self.compute_instance_limit}") + else: + self.size = new_target + + print(f"New MIG target size: {self.size}") + + if self.debug > 1: + print("MIG Information:") + print(responseGroupInfo) + + if self.size == current_target: + if current_target == 0: + print("Queue is empty") + print("Running correct number of VMs to handle queue") + exit() + + if self.size < current_target: + print("Scaling down. Looking for nodes that can be shut down") + + if self.debug > 1: + print("Compute node busy status:") + for node in idle_nodes: + print(node) + + # Shut down idle nodes up to our calculated limit + nodes_to_delete = list(idle_nodes.keys())[0:current_target-self.size] + for node in nodes_to_delete: + print(f"Attempting to delete: {node.rsplit('/',1)[-1]}") + respDel = self.deleteFromMig(nodes_to_delete) + + if self.debug > 1: + print("Scaling down complete") + + if self.size > current_target: + print( + "Scaling up. Need to increase number of instances to " + str(self.size) + ) + # Request to resize + request = self.instanceGroupManagers.resize( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + size=self.size, + ) + response = request.execute() + if self.debug > 1: + print("Requesting to increase MIG size") + pprint(response) + print("Scaling up complete") + + +def main(): + + scaler = AutoScaler(args.mz) + + # Project ID + scaler.project = args.p # Ex:'slurm-var-demo' + + # Name of the zone where the managed instance group is located + scaler.zone = args.z # Ex: 'us-central1-f' + + # Name of the region where the managed instance group is located + scaler.region = args.r # Ex: 'us-central1' + + # The name of the managed instance group. + scaler.instance_group_manager = args.g # Ex: 'condor-compute-igm' + + # Default number of cores per instance, will be replaced with actual value + scaler.cores_per_node = 4 + + # Default number of running instances that the managed instance group should maintain at any given time. This number will go up and down based on the load (number of jobs in the queue) + scaler.size = 0 + + scaler.compute_instance_min_idle = args.i + + # Dry run: : 0, run scaling; 1, only provide info. + scaler.dryrun = args.d > 0 + + # Debug level: 1-print debug information, 2 - print detail debug information + scaler.debug = 0 + if args.v: + scaler.debug = args.v + + # Limit for the maximum number of compute instance. If zero (default setting), no limit will be enforced by the script + scaler.compute_instance_limit = 0 + if args.c: + scaler.compute_instance_limit = abs(args.c) + + scaler.scale() + + +if __name__ == "__main__": + main() diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml new file mode 100644 index 0000000000..db989f9d40 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml @@ -0,0 +1,46 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Install but do not activate HTCondor autoscaler + become: true + hosts: localhost + tasks: + - name: Install Python 3 pip + ansible.builtin.package: + name: python3-pip + state: present + - name: Create virtual environment for HTCondor autoscaler + ansible.builtin.pip: + name: pip + version: 21.3.1 # last Python 3.6-compatible release + virtualenv: /usr/local/htcondor + virtualenv_command: /usr/bin/python3 -m venv + - name: Install latest setuptools + ansible.builtin.pip: + name: setuptools + version: 59.6.0 # last Python 3.6-compatible release + virtualenv: /usr/local/htcondor + virtualenv_command: /usr/bin/python3 -m venv + - name: Install HTCondor autoscaler dependencies + with_items: + - oauth2client + - google-api-python-client + - absl-py + - htcondor + ansible.builtin.pip: + name: "{{ item }}" + state: present # rely on pip resolver to pick latest compatible releases + virtualenv: /usr/local/htcondor + virtualenv_command: /usr/bin/python3 -m venv diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml new file mode 100644 index 0000000000..4d3abbbfd6 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml @@ -0,0 +1,94 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The instructions for installing HTCondor may change with time, although we +# anticipate that they will stay fixed for the 23.0 releases. Find up-to-date +# recommendations at: +## https://htcondor.readthedocs.io/en/latest/getting-htcondor/from-our-repositories.html + +--- +- name: Ensure HTCondor is installed + hosts: all + vars: + enable_docker: true + htcondor_key: https://research.cs.wisc.edu/htcondor/repo/keys/HTCondor-23.0-Key + docker_key: https://download.docker.com/linux/centos/gpg + become: true + module_defaults: + ansible.builtin.yum: + lock_timeout: 300 + tasks: + - name: Enable EPEL repository + ansible.builtin.yum: + name: + - epel-release + - name: Directly install RPM verification keys + ansible.builtin.rpm_key: + state: present + key: "{{ item }}" + loop: + - "{{ htcondor_key }}" + - "{{ docker_key }}" + register: key_install + retries: 10 + delay: 60 + until: key_install is success + - name: Enable HTCondor LTS Release repository + ansible.builtin.yum_repository: + name: htcondor-feature + description: HTCondor LTS Release (23.0) + file: htcondor + baseurl: https://research.cs.wisc.edu/htcondor/repo/23.0/el$releasever/$basearch/release + gpgkey: "{{ htcondor_key }}" + gpgcheck: true + repo_gpgcheck: true + priority: "90" + - name: Install HTCondor + ansible.builtin.yum: + name: condor-{{ condor_version | default("23.*") | string }} + state: present + - name: Ensure token directory + ansible.builtin.file: + path: /etc/condor/tokens.d + mode: 0700 + owner: root + group: root + - name: Install Docker and configure HTCondor to use it + when: enable_docker | bool # allows string to be passed at CLI + block: + - name: Setup Docker repo + ansible.builtin.yum_repository: + name: docker-ce-stable + description: Docker CE Stable - $basearch + baseurl: https://download.docker.com/linux/centos/$releasever/$basearch/stable + enabled: yes + gpgcheck: yes + gpgkey: "{{ docker_key }}" + - name: Install Docker + ansible.builtin.yum: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-compose-plugin + - name: Enable Docker + ansible.builtin.service: + name: docker + state: started + enabled: true + - name: Add condor to docker group + ansible.builtin.user: + name: condor + groups: docker + append: yes diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/main.tf new file mode 100644 index 0000000000..0853e035f4 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/main.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + runners = [ + { + "type" = "ansible-local" + "source" = "${path.module}/files/install-htcondor.yaml" + "destination" = "install-htcondor.yaml" + "args" = join(" ", [ + "-e enable_docker=${var.enable_docker}", + "-e condor_version=${var.condor_version}", + ]) + }, + { + "type" = "ansible-local" + "content" = file("${path.module}/files/install-htcondor-autoscaler-deps.yml") + "destination" = "install-htcondor-autoscaler-deps.yml" + }, + { + "type" = "data" + "content" = file("${path.module}/files/autoscaler.py") + "destination" = "/usr/local/htcondor/bin/autoscaler.py" + }, + ] + + install_htcondor_ps1 = templatefile( + "${path.module}/templates/install-htcondor.ps1.tftpl", { + condor_version = var.condor_version, + http_proxy = var.http_proxy, + python_windows_installer_url = var.python_windows_installer_url, + }) + + required_apis = [ + "compute.googleapis.com", + "secretmanager.googleapis.com", + ] +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf new file mode 100644 index 0000000000..c7951737ff --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "runners" { + description = "Runner to install HTCondor using startup-scripts" + value = local.runners +} + +output "windows_startup_ps1" { + description = "Windows PowerShell script to install HTCondor" + value = local.install_htcondor_ps1 +} + +output "gcp_service_list" { + description = "Google Cloud APIs required by HTCondor" + value = local.required_apis +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl new file mode 100644 index 0000000000..7492da3c12 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl @@ -0,0 +1,59 @@ +#Requires -RunAsAdministrator + +# Windows 2016 needs forced upgrade to TLS 1.2 +[Net.ServicePointManager]::SecurityProtocol = 'Tls12' + +# important for catching exception in Invoke-WebRequest +Set-StrictMode -Version latest +$ErrorActionPreference = 'Stop' + +%{ if http_proxy != "" ~} +[System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy("${http_proxy}") +%{ endif ~} + +# do not show progress bar when running Invoke-WebRequest +$ProgressPreference = 'SilentlyContinue' + +# download C Runtime DLL necessary for HTCondor installer +$runtime_installer = 'C:\vc_redist.x64.exe' +Invoke-WebRequest https://aka.ms/vs/17/release/vc_redist.x64.exe -OutFile "$runtime_installer" +Start-Process -FilePath "$runtime_installer" -Wait -ArgumentList "/norestart /quiet /log c:\vc_redist_log.txt" +Remove-Item "$runtime_installer" + +# download HTCondor installer +$htcondor_installer = 'C:\htcondor.msi' +%{ if condor_version == "23.*" } +Invoke-WebRequest https://research.cs.wisc.edu/htcondor/tarball/23.0/current/condor-Windows-x64.msi -OutFile "$htcondor_installer" +%{ else ~} +Invoke-WebRequest https://research.cs.wisc.edu/htcondor/tarball/23.0/${condor_version}/release/condor-${condor_version}-Windows-x64.msi -OutFile "$htcondor_installer" +%{ endif ~} +$args='/qn /l* condor-install-log.txt /i' +$args=$args + " $htcondor_installer" +$args=$args + ' NEWPOOL="N"' +$args=$args + ' RUNJOBS="N"' +$args=$args + ' SUBMITJOBS="N"' +$args=$args + ' INSTALLDIR="C:\Condor"' +Start-Process "msiexec.exe" -Wait -ArgumentList "$args" +Remove-Item "$htcondor_installer" + +# do not start HTCondor on boot by default. Allow startup script to download +# configuration first and then start HTCondor +Set-Service -StartupType Manual condor + +# remove settings from condor_config that we want to override in configuration step +Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^CONDOR_HOST' -NotMatch) +Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^INSTALL_USER' -NotMatch) +Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^DAEMON_LIST' -NotMatch) +Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^use SECURITY' -NotMatch) + +# install Python so that custom ClassAd hooks can execute +$python_installer = 'C:\python-installer.exe' +Invoke-WebRequest -Uri "${python_windows_installer_url}" -OutFile "$python_installer" +Start-Process -FilePath "$python_installer" -Wait -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1 Include_test=0' +%{ if http_proxy == "" ~} +Start-Process "py.exe" -Wait -ArgumentList "-3.11 -m pip install --no-warn-script-location requests" +%{ else ~} +Start-Process "py.exe" -Wait -ArgumentList "-3.11 -m pip install --proxy ${http_proxy} --no-warn-script-location requests" +%{ endif ~} +Invoke-WebRequest -Uri "https://raw.githubusercontent.com/htcondor/htcondor/main/src/condor_scripts/common-cloud-attributes-google.py" -OutFile "C:\Condor\bin\common-cloud-attributes-google.py" +Remove-Item "$python_installer" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/variables.tf new file mode 100644 index 0000000000..1afdf4e0eb --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/variables.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "enable_docker" { + description = "Install and enable docker daemon alongside HTCondor" + type = bool + default = true +} + +variable "condor_version" { + description = "Yum/DNF-compatible version string; leave unset to use latest 23.0 LTS release (examples: \"23.0.0\",\"23.*\"))" + type = string + default = "23.*" + + validation { + error_message = "var.condor_version must be set to \"23.*\" for latest 23.0 release or to a specific \"23.0.y\" release." + condition = var.condor_version == "23.*" || ( + length(split(".", var.condor_version)) == 3 && alltrue([ + for v in split(".", var.condor_version) : can(tonumber(v)) + ]) && split(".", var.condor_version)[0] == "23" + && split(".", var.condor_version)[1] == "0" + ) + } +} + +variable "http_proxy" { + description = "Set system default web (http and https) proxy for Windows HTCondor installation" + type = string + default = "" + nullable = false +} + +variable "python_windows_installer_url" { + description = "URL of Python installer for Windows" + type = string + default = "https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe" + nullable = false +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/versions.tf new file mode 100644 index 0000000000..79b6fbde47 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/README.md new file mode 100644 index 0000000000..55c2fc7e4e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/README.md @@ -0,0 +1,116 @@ +## Description + +This module will create a startup-script runner that will execute Ramble commands. + +Ramble is a multi-platform experimentation framework capable of driving +software installation, acquiring input files, configuring experiments, and +extracting results. For more information about Ramble, see: +https://github.com/GoogleCloudPlatform/ramble + +This module outputs a startup script runner, which can be combined with other +startup script runners to execute a set of Ramble commands. + +Ramble makes extensive use of Spack. It must be installed with a Toolkit runner +generated by the [spack-setup module](../spack-setup/README.md) following the +[basic example](#basic-example) below. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +# Examples + +## Basic Example + +Below is a basic example of using this module. + +```yaml + - id: spack + source: community/modules/scripts/spack-setup + + - id: ramble-setup + source: community/modules/scripts/ramble-setup + + - id: ramble-execute + source: community/modules/scripts/ramble-execute + use: [spack, ramble-setup] + settings: + commands: + - ramble list +``` + +This example shows installing Spack and Ramble with their own modules +(spack-setup and ramble-setup respectively). Then the ramble-execute module +is added to simply list all applications Ramble knows about. + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [local](#requirement\_local) | >= 2.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [local](#provider\_local) | >= 2.0.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [local_file.debug_file_ansible_execute](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [commands](#input\_commands) | String of commands to run within this module | `string` | `null` | no | +| [data\_files](#input\_data\_files) | A list of files to be transferred prior to running commands.
It must specify one of 'source' (absolute local file path) or 'content' (string).
It must specify a 'destination' with absolute path where file should be placed. | `list(map(string))` | `[]` | no | +| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing spack scripts. | `string` | n/a | yes | +| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | n/a | yes | +| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | +| [log\_file](#input\_log\_file) | Log file to write output from Ramble execute steps into | `string` | `"/var/log/ramble-execute.log"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [ramble\_profile\_script\_path](#input\_ramble\_profile\_script\_path) | Path to the Ramble profile.d script. Created by an instance of ramble-setup.
Can be defined explicitly, or by chaining an instance of a ramble-setup module
through a `use` setting. | `string` | n/a | yes | +| [ramble\_runner](#input\_ramble\_runner) | Runner from previous ramble-setup or ramble-execute to be chained with scripts generated by this module. | 
object({
    type        = string
    content     = string
    destination = string
  })
| n/a | yes | +| [region](#input\_region) | Region to place bucket containing spack scripts. | `string` | n/a | yes | +| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script.
Can be defined explicitly, or by chaining an instance of a spack-setup module
through a `use` setting.
Defaults to /etc/profile.d/spack.sh if not set. | `string` | `"/etc/profile.d/spack.sh"` | no | +| [system\_user\_name](#input\_system\_user\_name) | Name of the system user used to execute commands. Generally passed from the ramble-setup module. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [controller\_startup\_script](#output\_controller\_startup\_script) | Ramble startup script, duplicate for SLURM controller. | +| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for ramble, to be reused by ramble-execute module. | +| [ramble\_profile\_script\_path](#output\_ramble\_profile\_script\_path) | Path to Ramble profile script. | +| [ramble\_runner](#output\_ramble\_runner) | Runner to execute Ramble commands using an ansible playbook. The startup-script module
will automatically handle installation of ansible. | +| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to Spack profile script. | +| [startup\_script](#output\_startup\_script) | Ramble startup script. | +| [system\_user\_name](#output\_system\_user\_name) | The system user used to execute commands. | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/main.tf new file mode 100644 index 0000000000..7ef0b029e3 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/main.tf @@ -0,0 +1,71 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "ramble-execute", ghpc_role = "scripts" }) +} + +locals { + commands_content = var.commands == null ? "echo 'no ramble commands provided'" : indent(4, yamlencode(var.commands)) + + execute_contents = templatefile( + "${path.module}/templates/ramble_execute.yml.tpl", + { + pre_script = "if [ -f ${var.spack_profile_script_path} ]; then . ${var.spack_profile_script_path}; fi; . ${var.ramble_profile_script_path}" + log_file = var.log_file + commands = local.commands_content + system_user_name = var.system_user_name + } + ) + + data_runners = [for data_file in var.data_files : merge(data_file, { type = "data" })] + + execute_md5 = substr(md5(local.execute_contents), 0, 4) + execute_runner = { + type = "ansible-local" + content = local.execute_contents + destination = "ramble_execute_${local.execute_md5}.yml" + } + + previous_runners = var.ramble_runner != null ? [var.ramble_runner] : [] + runners = concat(local.previous_runners, local.data_runners, [local.execute_runner]) + + # Destinations should be unique while also being known at time of apply + combined_unique_string = join("\n", [for runner in local.runners : runner["destination"]]) + combined_md5 = substr(md5(local.combined_unique_string), 0, 4) + combined_runner = { + type = "shell" + content = module.startup_script.startup_script + destination = "combined_install_ramble_${local.combined_md5}.sh" + } +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.runners + gcs_bucket_path = var.gcs_bucket_path +} + +resource "local_file" "debug_file_ansible_execute" { + content = local.execute_contents + filename = "${path.module}/debug_execute_${local.execute_md5}.yml" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf new file mode 100644 index 0000000000..4e6c3a44d8 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf @@ -0,0 +1,53 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "Ramble startup script." + value = module.startup_script.startup_script +} + +output "controller_startup_script" { + description = "Ramble startup script, duplicate for SLURM controller." + value = module.startup_script.startup_script +} + +output "ramble_runner" { + description = <<-EOT + Runner to execute Ramble commands using an ansible playbook. The startup-script module + will automatically handle installation of ansible. + EOT + value = local.combined_runner +} + +output "gcs_bucket_path" { + description = "Bucket containing the startup scripts for ramble, to be reused by ramble-execute module." + value = var.gcs_bucket_path +} + +output "spack_profile_script_path" { + description = "Path to Spack profile script." + value = var.spack_profile_script_path +} + +output "ramble_profile_script_path" { + description = "Path to Ramble profile script." + value = var.ramble_profile_script_path +} + +output "system_user_name" { + description = "The system user used to execute commands." + value = var.system_user_name +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl new file mode 100644 index 0000000000..0e98f3aa2c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl @@ -0,0 +1,59 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Execute Commands + hosts: localhost + vars: + pre_script: ${pre_script} + log_file: ${log_file} + commands: ${commands} + system_user_name: ${system_user_name} + tasks: + - name: Execute command block + block: + - name: Print commands to be executed + ansible.builtin.debug: + msg: "{{ commands.split('\n') | ansible.builtin.to_nice_yaml }}" + + - name: Streaming log info + ansible.builtin.debug: + msg: | + Logs from commands will not be printed here until success (or failure) + Streaming logs can be found at {{ log_file }} + + - name: Ensure user can write to log file + ansible.builtin.file: + path: "{{ log_file }}" + state: touch + owner: "{{ system_user_name }}" + + - name: Execute commands + ansible.builtin.shell: | + set -eo pipefail + { + {{ pre_script }} + echo " === Starting commands ===" + {{ commands }} + echo " === Finished commands ===" + } 2>&1 | tee -a {{ log_file }} + args: + executable: /bin/bash + register: output + become: true + become_user: "{{ system_user_name }}" + + always: + - name: Print commands output + ansible.builtin.debug: + var: output.stdout_lines diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/variables.tf new file mode 100644 index 0000000000..ec67228df5 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/variables.tf @@ -0,0 +1,114 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created." + type = string +} + +variable "deployment_name" { + description = "Name of deployment, used to name bucket containing spack scripts." + type = string +} + +variable "region" { + description = "Region to place bucket containing spack scripts." + type = string +} + +variable "labels" { + description = "Key-value pairs of labels to be added to created resources." + type = map(string) +} + +variable "log_file" { + description = "Log file to write output from Ramble execute steps into" + default = "/var/log/ramble-execute.log" + type = string +} + +variable "data_files" { + description = <<-EOT + A list of files to be transferred prior to running commands. + It must specify one of 'source' (absolute local file path) or 'content' (string). + It must specify a 'destination' with absolute path where file should be placed. + EOT + type = list(map(string)) + default = [] + validation { + condition = alltrue([for r in var.data_files : substr(r["destination"], 0, 1) == "/"]) + error_message = "All destinations must be absolute paths and start with '/'." + } + validation { + condition = alltrue([ + for r in var.data_files : + can(r["content"]) != can(r["source"]) + ]) + error_message = "A data_file must specify either 'content' or 'source', but never both." + } + validation { + condition = alltrue([ + for r in var.data_files : + lookup(r, "content", lookup(r, "source", null)) != null + ]) + error_message = "A data_file must specify a non-null 'content' or 'source'." + } +} + +variable "commands" { + description = "String of commands to run within this module" + default = null + type = string +} + +variable "ramble_runner" { + description = "Runner from previous ramble-setup or ramble-execute to be chained with scripts generated by this module." + type = object({ + type = string + content = string + destination = string + }) +} + +variable "system_user_name" { + description = "Name of the system user used to execute commands. Generally passed from the ramble-setup module." + type = string +} + +variable "gcs_bucket_path" { + description = "The GCS path for storage bucket and the object, starting with `gs://`." + type = string +} + +variable "spack_profile_script_path" { + description = <<-EOT + Path to the Spack profile.d script. + Can be defined explicitly, or by chaining an instance of a spack-setup module + through a `use` setting. + Defaults to /etc/profile.d/spack.sh if not set. + EOT + type = string + default = "/etc/profile.d/spack.sh" +} + +variable "ramble_profile_script_path" { + description = <<-EOT + Path to the Ramble profile.d script. Created by an instance of ramble-setup. + Can be defined explicitly, or by chaining an instance of a ramble-setup module + through a `use` setting. + EOT + type = string +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/versions.tf new file mode 100644 index 0000000000..9b23317323 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/versions.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0.0" + required_providers { + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/README.md new file mode 100644 index 0000000000..9891088105 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/README.md @@ -0,0 +1,128 @@ +## Description + +This module will create a set of startup-script runners that will setup Ramble, +and install Rambleā€™s dependencies. + +Ramble is a multi-platform experimentation framework capable of driving +software installation, acquiring input files, configuring experiments, and +extracting results. For more information about ramble, see: +https://github.com/GoogleCloudPlatform/ramble + +This module outputs two startup script runners, which can be added to startup +scripts to setup, ramble and its dependencies. + +For this module to be completely functional, it depends on a spack +installation. For more information, see Cluster-Toolkitā€™s Spack module. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +# Examples + +## Basic Example + +```yaml +- id: ramble-setup + source: community/modules/scripts/ramble-setup +``` + +This example simply installs ramble on a VM. + +## Full Example + +```yaml +- id: ramble-setup + source: community/modules/scripts/ramble-setup + settings: + install_dir: /ramble + ramble_url: https://github.com/GoogleCloudPlatform/ramble + ramble_ref: v0.2.1 + log_file: /var/log/ramble.log + chown_owner: ā€œownerā€ + chgrp_group: ā€œuser_groupā€ + chmod_mode: ā€œa+rā€ +``` + +This example simply installs ramble into a VM at the location `/ramble`, checks +out the v0.2.1 tag, changes the owner and group to ā€œownerā€ and ā€œuser_groupā€, +and chmodā€™s the clone to make it world readable. + +Also see a more complete [Ramble example blueprint](../../../examples/ramble.yaml). + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [google](#requirement\_google) | >= 4.42 | +| [local](#requirement\_local) | >= 2.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [local](#provider\_local) | >= 2.0.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | +| [local_file.debug_file_shell_install](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [chmod\_mode](#input\_chmod\_mode) | Mode to chmod the Ramble clone to. Defaults to `""` (i.e. do not modify).
For usage information see:
https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode | `string` | `""` | no | +| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing startup script. | `string` | n/a | yes | +| [install\_dir](#input\_install\_dir) | Destination directory of installation of Ramble. | `string` | `"/apps/ramble"` | no | +| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [ramble\_profile\_script\_path](#input\_ramble\_profile\_script\_path) | Path to the Ramble profile.d script. Created by this module | `string` | `"/etc/profile.d/ramble.sh"` | no | +| [ramble\_ref](#input\_ramble\_ref) | Git ref to checkout for Ramble. | `string` | `"develop"` | no | +| [ramble\_url](#input\_ramble\_url) | URL for Ramble repository to clone. | `string` | `"https://github.com/GoogleCloudPlatform/ramble"` | no | +| [ramble\_virtualenv\_path](#input\_ramble\_virtualenv\_path) | Virtual environment path in which to install Ramble Python interpreter and other dependencies | `string` | `"/usr/local/ramble-python"` | no | +| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | +| [system\_user\_gid](#input\_system\_user\_gid) | GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary. | `number` | `1104762904` | no | +| [system\_user\_name](#input\_system\_user\_name) | Name of system user that will perform installation of Ramble. It will be created if it does not exist. | `string` | `"ramble"` | no | +| [system\_user\_uid](#input\_system\_user\_uid) | UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary. | `number` | `1104762904` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [controller\_startup\_script](#output\_controller\_startup\_script) | Ramble installation script, duplicate for SLURM controller. | +| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for Ramble, to be reused by ramble-execute module. | +| [ramble\_path](#output\_ramble\_path) | Location ramble is installed into. | +| [ramble\_profile\_script\_path](#output\_ramble\_profile\_script\_path) | Path to Ramble profile script. | +| [ramble\_ref](#output\_ramble\_ref) | Git ref the ramble install is checked out to use | +| [ramble\_runner](#output\_ramble\_runner) | Runner to be used with startup-script module or passed to ramble-execute module.
- installs Ramble dependencies
- installs Ramble
- generates profile.d script to enable access to Ramble
This is safe to run in parallel by multiple machines. | +| [startup\_script](#output\_startup\_script) | Ramble installation script. | +| [system\_user\_name](#output\_system\_user\_name) | The system user used to install Ramble. It can be reused by ramble-execute module to execute Ramble commands. | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/main.tf new file mode 100644 index 0000000000..4389af7d33 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/main.tf @@ -0,0 +1,113 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "ramble-setup", ghpc_role = "scripts" }) +} + +locals { + profile_script = <<-EOF + if [ -f ${var.install_dir}/share/ramble/setup-env.sh ]; then + test -t 1 && echo "** Ramble's python virtualenv (/usr/local/ramble-python) is activated. Call 'deactivate' to deactivate." + VIRTUAL_ENV_DISABLE_PROMPT=1 . ${var.ramble_virtualenv_path}/bin/activate + . ${var.install_dir}/share/ramble/setup-env.sh + fi + EOF + + script_content = templatefile( + "${path.module}/templates/ramble_setup.yml.tftpl", + { + sw_name = "ramble" + profile_script = indent(4, yamlencode(local.profile_script)) + install_dir = var.install_dir + git_url = var.ramble_url + git_ref = var.ramble_ref + chmod_mode = var.chmod_mode + system_user_name = var.system_user_name + system_user_uid = var.system_user_uid + system_user_gid = var.system_user_gid + finalize_setup_script = "echo 'no finalize setup script'" + profile_script_path = var.ramble_profile_script_path + } + ) + + install_ramble_deps_runner = { + "type" = "ansible-local" + "source" = "${path.module}/scripts/install_ramble_deps.yml" + "destination" = "install_ramble_deps.yml" + "args" = "-e virtualenv_path=${var.ramble_virtualenv_path}" + } + + python_reqs_content = templatefile( + "${path.module}/templates/install_ramble_python_deps.yml.tftpl", + { + install_dir = var.install_dir + virtualenv_path = var.ramble_virtualenv_path + } + ) + + python_reqs_runner = { + "type" = "ansible-local" + "content" = local.python_reqs_content + "destination" = "install_ramble_reqs.yml" + } + + install_ramble_runner = { + "type" = "ansible-local" + "content" = local.script_content + "destination" = "install_ramble.yml" + } + + bucket_md5 = substr(md5("${var.project_id}.${var.deployment_name}"), 0, 8) + # Max bucket name length is 63, so truncate deployment_name if necessary. + # The string "-ramble-scripts-" is 16 characters and bucket_md5 is 8 characters, + # leaving 63-16-8=39 chars for deployment_name. + bucket_name = "${substr(var.deployment_name, 0, 39)}-ramble-scripts-${local.bucket_md5}" + runners = [local.install_ramble_deps_runner, local.install_ramble_runner, local.python_reqs_runner] + + combined_runner = { + "type" = "shell" + "content" = module.startup_script.startup_script + "destination" = "ramble-install-and-setup.sh" + } + +} + +resource "google_storage_bucket" "bucket" { + project = var.project_id + name = local.bucket_name + uniform_bucket_level_access = true + location = var.region + storage_class = "REGIONAL" + labels = local.labels +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.runners + gcs_bucket_path = "gs://${google_storage_bucket.bucket.name}" +} + +resource "local_file" "debug_file_shell_install" { + content = local.script_content + filename = "${path.module}/debug_install.yml" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf new file mode 100644 index 0000000000..e587470eac --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf @@ -0,0 +1,61 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "Ramble installation script." + value = module.startup_script.startup_script +} + +output "controller_startup_script" { + description = "Ramble installation script, duplicate for SLURM controller." + value = module.startup_script.startup_script +} + +output "ramble_runner" { + description = <<-EOT + Runner to be used with startup-script module or passed to ramble-execute module. + - installs Ramble dependencies + - installs Ramble + - generates profile.d script to enable access to Ramble + This is safe to run in parallel by multiple machines. + EOT + value = local.combined_runner +} + +output "ramble_path" { + description = "Location ramble is installed into." + value = var.install_dir +} + +output "ramble_ref" { + description = "Git ref the ramble install is checked out to use" + value = var.ramble_ref +} + +output "gcs_bucket_path" { + description = "Bucket containing the startup scripts for Ramble, to be reused by ramble-execute module." + value = "gs://${google_storage_bucket.bucket.name}" +} + +output "ramble_profile_script_path" { + description = "Path to Ramble profile script." + value = var.ramble_profile_script_path +} + +output "system_user_name" { + description = "The system user used to install Ramble. It can be reused by ramble-execute module to execute Ramble commands." + value = var.system_user_name +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml new file mode 100644 index 0000000000..b7905bbe9e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml @@ -0,0 +1,50 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Create python virtual env for a tool + become: yes + hosts: localhost + vars: + virtualenv_path: ${virtualenv_path} + tasks: + - name: Install dependencies through system package manager + ansible.builtin.package: + name: + - python3 + - python3-pip + - git + register: package + changed_when: package.changed + retries: 5 + delay: 10 + until: package is success + + - name: Create virtualenv for tool + # Python 3.6 is minimum we wish to support due to ease of installation on + # CentOS 7 and Rocky Linux 8. pip 21.3.1 is the *maximum* version of pip + # supported by 3.6. Additionally, recent versions of pip are necessary for + # proper dependency resolution of real-world problems with google-cloud-* + # (and third-party) Python packages (20.3+ probably effective minimum). + ansible.builtin.pip: + name: pip>=21.3.1 + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: /usr/bin/python3 -m venv + + - name: Add google-cloud-storage to virtualenv + ansible.builtin.pip: + name: google-cloud-storage + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: /usr/bin/python3 -m venv diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl new file mode 100644 index 0000000000..ea14780a58 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl @@ -0,0 +1,28 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Install Python Requirements + hosts: localhost + vars: + install_dir: ${install_dir} + virtualenv_path: ${virtualenv_path} + tasks: + + - name: Install dependencies + ansible.builtin.pip: + requirements: "{{ install_dir }}/requirements.txt" + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: /usr/bin/python3 -m venv diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl new file mode 100644 index 0000000000..ca48a5afa0 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl @@ -0,0 +1,157 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Install Software + hosts: localhost + vars: + sw_name: ${sw_name} + profile_script: ${profile_script} + install_dir: ${install_dir} + git_url: ${git_url} + git_ref: ${git_ref} + chmod_mode: ${chmod_mode} + system_user_name: ${system_user_name} + system_user_uid: ${system_user_uid} + system_user_gid: ${system_user_gid} + finalize_setup_script: ${finalize_setup_script} + profile_script_path: ${profile_script_path} + tasks: + - name: Print software name + ansible.builtin.debug: + msg: "Running installation for software: {{ sw_name }}" + + - name: Add profile script for software + ansible.builtin.copy: + dest: "{{ profile_script_path }}" + mode: '0644' + content: "{{ profile_script }}" + when: profile_script + + - name: Look up user to use for install + block: + + - name: Check if user already exists + ansible.builtin.getent: + database: passwd + key: "{{ system_user_name }}" + + - name: Look up existing user details + ansible.builtin.user: + name: "{{ system_user_name }}" + register: system_user + + rescue: + - name: User did not exist, create group for system user + ansible.builtin.group: + name: "{{ system_user_name }}" + gid: "{{ system_user_gid }}" + system: true + register: system_group + + - name: Create system user + ansible.builtin.user: + name: "{{ system_user_name }}" + comment: "{{ sw_name }} installation" + uid: "{{ system_user_uid }}" + group: "{{ system_group.name }}" + system: true + register: system_user + + - name: Create parent of install directory + ansible.builtin.file: + path: "{{ install_dir | dirname }}" + state: directory + + - name: Set lock dir + ansible.builtin.set_fact: + lock_dir: "{{ install_dir | dirname }}/.install_{{ sw_name }}_lock" + + - name: Acquire lock + ansible.builtin.command: + mkdir "{{ lock_dir }}" + register: lock_out + changed_when: lock_out.rc == 0 + failed_when: false + + - name: Add hostname to lock_dir + ansible.builtin.file: + path: "{{ lock_dir }}/{{ ansible_hostname }}" + state: touch + when: lock_out.rc == 0 + + - name: Clone branch or tag into installation directory + ansible.builtin.command: git clone --branch {{ git_ref }} {{ git_url }} {{ install_dir }} + failed_when: false + register: clone_res + when: lock_out.rc == 0 + + - name: Clone commit hash into installation directory + ansible.builtin.command: "{{ item }}" + with_items: + - git clone {{ git_url }} {{ install_dir }} + - git -C {{ install_dir }} checkout {{ git_ref }} + when: lock_out.rc == 0 and clone_res.rc != 0 + + - name: Transfer ownership to system user + ansible.builtin.file: + path: "{{ install_dir }}" + owner: "{{ system_user.name }}" + group: "{{ system_user.group }}" + recurse: true + follow: false + when: lock_out.rc == 0 + + - name: Finalize setup + ansible.builtin.shell: "{{ finalize_setup_script }}" + when: lock_out.rc == 0 and finalize_setup_script + become: true + become_user: "{{ system_user.name }}" + + - name: Apply chmod + ansible.builtin.file: + path: "{{ install_dir }}" + mode: "{{ chmod_mode | default(omit, true) }}" + recurse: true + follow: false + when: (lock_out.rc == 0) and (chmod_mode != None) + + - name: Release lock + ansible.builtin.file: + path: "{{ lock_dir }}/done" + state: touch + when: lock_out.rc == 0 + + - name: Wait for lock + block: + - name: Wait for lock + ansible.builtin.wait_for: + path: "{{ lock_dir }}/done" + state: present + timeout: 600 + sleep: 10 + when: lock_out.rc != 0 + + rescue: + - name: Timed out on waiting for lock, get lock directory contents + ansible.builtin.find: + paths: "{{ lock_dir }}" + register: lock_dir_contents + + - name: Print lock directory contents, it should contain name of host that is holding lock + ansible.builtin.debug: + msg: "{{ lock_dir_contents.files|map(attribute='path')|map('basename')|list }}" + + - name: Failed to get lock + ansible.builtin.fail: + msg: "Timeout waiting on lock for ${sw_name}, exiting" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/variables.tf new file mode 100644 index 0000000000..0d3a8eed05 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/variables.tf @@ -0,0 +1,97 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created." + type = string +} + +variable "install_dir" { + description = "Destination directory of installation of Ramble." + default = "/apps/ramble" + type = string +} + +variable "ramble_url" { + description = "URL for Ramble repository to clone." + default = "https://github.com/GoogleCloudPlatform/ramble" + type = string +} + +variable "ramble_ref" { + description = "Git ref to checkout for Ramble." + default = "develop" + type = string +} + +variable "chmod_mode" { + description = <<-EOT + Mode to chmod the Ramble clone to. Defaults to `""` (i.e. do not modify). + For usage information see: + https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode + EOT + default = "" + type = string + nullable = false +} + +variable "system_user_name" { + description = "Name of system user that will perform installation of Ramble. It will be created if it does not exist." + default = "ramble" + type = string + nullable = false +} + +variable "system_user_uid" { + description = "UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary." + default = 1104762904 + type = number + nullable = false +} + +variable "system_user_gid" { + description = "GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary." + default = 1104762904 + type = number + nullable = false +} + +variable "ramble_virtualenv_path" { + description = "Virtual environment path in which to install Ramble Python interpreter and other dependencies" + default = "/usr/local/ramble-python" + type = string +} + +variable "deployment_name" { + description = "Name of deployment, used to name bucket containing startup script." + type = string +} + +variable "region" { + description = "Region to place bucket containing startup script." + type = string +} + +variable "labels" { + description = "Key-value pairs of labels to be added to created resources." + type = map(string) +} + +variable "ramble_profile_script_path" { + description = "Path to the Ramble profile.d script. Created by this module" + type = string + default = "/etc/profile.d/ramble.sh" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/versions.tf new file mode 100644 index 0000000000..936b4a5b80 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/versions.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/README.md new file mode 100644 index 0000000000..8cbb75fb42 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/README.md @@ -0,0 +1,141 @@ +## Description + +This module creates a script that defines a software build using Spack and +performs any additional customization to a Spack installation. + +There are two main variable inputs that can be used to define a Spack build: +`data_files` and `commands`. + +- `data_files`: Any files specified will be transferred to the machine running + outputted script. Data file `content` can be defined inline in the blueprint + or can point to a `source`, an absolute local path of a file. This can be used + to transfer environment definition files, config definition files, GPG keys, + or software licenses. `data_files` are transferred before `commands` are run. +- `commands`: A script that is run. This can be used to perform actions such as + installation of compilers & packages, environment creation, adding a build + cache, and modifying the spack configuration. + +## Example + +The `spack-execute` module should `use` a `spack-setup` module. This will +prepend the installation of Spack and its dependencies to the build. Then +`spack-execute` can be used by a module that takes `startup-script` as an input. + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: spack-build + source: community/modules/scripts/spack-execute + use: [spack-setup] + settings: + commands: | + spack install gcc@10.3.0 target=x86_64 + + - id: builder-vm + source: modules/compute/vm-instance + use: [network1, spack-build] +``` + +To see a full example of this module in use, see the [hpc-slurm-gromacs.yaml] example. + +[hpc-slurm-gromacs.yaml]: ../../../examples/hpc-slurm-gromacs.yaml + +### Using with `startup-script` module + +The `spack-runner` output can be used by the `startup-script` module. + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: spack-build + source: community/modules/scripts/spack-execute + use: [spack-setup] + settings: + commands: | + spack install gcc@10.3.0 target=x86_64 + + - id: startup-script + source: modules/scripts/startup-script + settings: + runners: + - $(spack-build.spack-runner) + - type: shell + destination: "my-script.sh" + content: echo 'hello world' + + - id: workstation + source: modules/compute/vm-instance + use: [network1, startup-script] +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [local](#requirement\_local) | >= 2.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [local](#provider\_local) | >= 2.0.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [local_file.debug_file_ansible_execute](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [commands](#input\_commands) | String of commands to run within this module | `string` | `null` | no | +| [data\_files](#input\_data\_files) | A list of files to be transferred prior to running commands.
It must specify one of 'source' (absolute local file path) or 'content' (string).
It must specify a 'destination' with absolute path where file should be placed. | `list(map(string))` | `[]` | no | +| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing spack scripts. | `string` | n/a | yes | +| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | n/a | yes | +| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | +| [log\_file](#input\_log\_file) | Defines the logfile that script output will be written to | `string` | `"/var/log/spack.log"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [region](#input\_region) | Region to place bucket containing spack scripts. | `string` | n/a | yes | +| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script. Created by an instance of spack-setup.
Can be defined explicitly, or by chaining an instance of a spack-setup module
through a `use` setting. | `string` | n/a | yes | +| [spack\_runner](#input\_spack\_runner) | Runner from previous spack-setup or spack-execute to be chained with scripts generated by this module. | 
object({
    type        = string
    content     = string
    destination = string
  })
| n/a | yes | +| [system\_user\_name](#input\_system\_user\_name) | Name of the system user used to execute commands. Generally passed from the spack-setup module. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [controller\_startup\_script](#output\_controller\_startup\_script) | Spack startup script, duplicate for SLURM controller. | +| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for spack, to be reused by spack-execute module. | +| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to the Spack profile.d script. | +| [spack\_runner](#output\_spack\_runner) | Single runner that combines scripts from this module and any previously chained spack-execute or spack-setup modules. | +| [startup\_script](#output\_startup\_script) | Spack startup script. | +| [system\_user\_name](#output\_system\_user\_name) | The system user used to execute commands. | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/main.tf new file mode 100644 index 0000000000..04ebcf7d49 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/main.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "spack-execute", ghpc_role = "scripts" }) +} + +locals { + commands_content = var.commands == null ? "echo 'no spack commands provided'" : indent(4, yamlencode(var.commands)) + + execute_contents = templatefile( + "${path.module}/templates/execute_commands.yml.tpl", + { + pre_script = ". ${var.spack_profile_script_path}" + log_file = var.log_file + commands = local.commands_content + system_user_name = var.system_user_name + } + ) + + data_runners = [for data_file in var.data_files : merge(data_file, { type = "data" })] + + execute_md5 = substr(md5(local.execute_contents), 0, 4) + execute_runner = { + type = "ansible-local" + content = local.execute_contents + destination = "spack_execute_${local.execute_md5}.yml" + } + + runners = concat([var.spack_runner], local.data_runners, [local.execute_runner]) + + # Destinations should be unique while also being known at time of apply + combined_unique_string = join("\n", [for runner in local.runners : runner["destination"]]) + combined_md5 = substr(md5(local.combined_unique_string), 0, 4) + combined_runner = { + type = "shell" + content = module.startup_script.startup_script + destination = "combined_install_spack_${local.combined_md5}.sh" + } +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.runners + gcs_bucket_path = var.gcs_bucket_path +} + +resource "local_file" "debug_file_ansible_execute" { + content = local.execute_contents + filename = "${path.module}/debug_execute_${local.execute_md5}.yml" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/outputs.tf new file mode 100644 index 0000000000..4a52532d51 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/outputs.tf @@ -0,0 +1,45 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "Spack startup script." + value = module.startup_script.startup_script +} + +output "controller_startup_script" { + description = "Spack startup script, duplicate for SLURM controller." + value = module.startup_script.startup_script +} + +output "spack_runner" { + description = "Single runner that combines scripts from this module and any previously chained spack-execute or spack-setup modules." + value = local.combined_runner +} + +output "gcs_bucket_path" { + description = "Bucket containing the startup scripts for spack, to be reused by spack-execute module." + value = var.gcs_bucket_path +} + +output "spack_profile_script_path" { + description = "Path to the Spack profile.d script." + value = var.spack_profile_script_path +} + +output "system_user_name" { + description = "The system user used to execute commands." + value = var.system_user_name +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl new file mode 100644 index 0000000000..0e98f3aa2c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl @@ -0,0 +1,59 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Execute Commands + hosts: localhost + vars: + pre_script: ${pre_script} + log_file: ${log_file} + commands: ${commands} + system_user_name: ${system_user_name} + tasks: + - name: Execute command block + block: + - name: Print commands to be executed + ansible.builtin.debug: + msg: "{{ commands.split('\n') | ansible.builtin.to_nice_yaml }}" + + - name: Streaming log info + ansible.builtin.debug: + msg: | + Logs from commands will not be printed here until success (or failure) + Streaming logs can be found at {{ log_file }} + + - name: Ensure user can write to log file + ansible.builtin.file: + path: "{{ log_file }}" + state: touch + owner: "{{ system_user_name }}" + + - name: Execute commands + ansible.builtin.shell: | + set -eo pipefail + { + {{ pre_script }} + echo " === Starting commands ===" + {{ commands }} + echo " === Finished commands ===" + } 2>&1 | tee -a {{ log_file }} + args: + executable: /bin/bash + register: output + become: true + become_user: "{{ system_user_name }}" + + always: + - name: Print commands output + ansible.builtin.debug: + var: output.stdout_lines diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/variables.tf new file mode 100644 index 0000000000..851cd1aed8 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/variables.tf @@ -0,0 +1,103 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created." + type = string +} + +variable "deployment_name" { + description = "Name of deployment, used to name bucket containing spack scripts." + type = string +} + +variable "region" { + description = "Region to place bucket containing spack scripts." + type = string +} + +variable "labels" { + description = "Key-value pairs of labels to be added to created resources." + type = map(string) +} + +variable "log_file" { + description = "Defines the logfile that script output will be written to" + default = "/var/log/spack.log" + type = string +} + +variable "data_files" { + description = <<-EOT + A list of files to be transferred prior to running commands. + It must specify one of 'source' (absolute local file path) or 'content' (string). + It must specify a 'destination' with absolute path where file should be placed. + EOT + type = list(map(string)) + default = [] + validation { + condition = alltrue([for r in var.data_files : substr(r["destination"], 0, 1) == "/"]) + error_message = "All destinations must be absolute paths and start with '/'." + } + validation { + condition = alltrue([ + for r in var.data_files : + can(r["content"]) != can(r["source"]) + ]) + error_message = "A data_file must specify either 'content' or 'source', but never both." + } + validation { + condition = alltrue([ + for r in var.data_files : + lookup(r, "content", lookup(r, "source", null)) != null + ]) + error_message = "A data_file must specify a non-null 'content' or 'source'." + } +} + +variable "commands" { + description = "String of commands to run within this module" + type = string + default = null +} + +variable "spack_runner" { + description = "Runner from previous spack-setup or spack-execute to be chained with scripts generated by this module." + type = object({ + type = string + content = string + destination = string + }) +} + +variable "system_user_name" { + description = "Name of the system user used to execute commands. Generally passed from the spack-setup module." + type = string +} + +variable "gcs_bucket_path" { + description = "The GCS path for storage bucket and the object, starting with `gs://`." + type = string +} + +variable "spack_profile_script_path" { + description = <<-EOT + Path to the Spack profile.d script. Created by an instance of spack-setup. + Can be defined explicitly, or by chaining an instance of a spack-setup module + through a `use` setting. + EOT + type = string +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/versions.tf new file mode 100644 index 0000000000..09583c3d43 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/versions.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.0.0" + required_providers { + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/README.md new file mode 100644 index 0000000000..01d3e6d389 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/README.md @@ -0,0 +1,382 @@ +## Description + +This module can be used to setup and install Spack on a VM. To actually run +Spack commands to install other software use the +[spack-execute](../spack-execute/) module. + +This module generates a script that performs the following: + +1. Install system dependencies needed for Spack +1. Clone Spack into a predefined directory +1. Check out a specific version of Spack + +There are several options on how to consume the outputs of this module: + +> [!IMPORTANT] +> Breaking changes between after v1.21.0. `spack-install` module replaced by +> `spack-setup` and `spack-execute` modules. +> [Details Below](#deprecations-and-breaking-changes) + +## Examples + +### `use` `spack-setup` with `spack-execute` + +This will prepend the `spack-setup` script to the `spack-execute` commands. + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: spack-build + source: community/modules/scripts/spack-execute + use: [spack-setup] + settings: + commands: | + spack install gcc@10.3.0 target=x86_64 + + - id: builder + source: modules/compute/vm-instance + use: [network1, spack-build] +``` + +### `use` `spack-setup` with `vm-instance` or Slurm module + +This will run `spack-setup` scripts on the downstream compute resource. + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: spack-installer + source: modules/compute/vm-instance + use: [network1, spack-setup] +``` + +OR + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + use: [network1, partition1, spack-setup] +``` + +### Build `starup-script` with `spack-runner` output + +This will use the generated `spack-setup` script as one step in `startup-script`. + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + + - id: startup-script + source: modules/scripts/startup-script + settings: + runners: + - $(spack-setup.spack-runner) + - type: shell + destination: "my-script.sh" + content: echo 'hello world' + + - id: workstation + source: modules/compute/vm-instance + use: [network1, startup-script] +``` + +To see a full example of this module in use, see the [hpc-slurm-gromacs.yaml] example. + +[hpc-slurm-gromacs.yaml]: ../../../examples/hpc-slurm-gromacs.yaml + +## Environment Setup + +### Activating Spack + +[Spack installation] produces a setup script that adds `spack` to your `PATH` as +well as some other command-line integration tools. This script can be found at +`/share/spack/setup-env.sh`. This script will be automatically +added to bash startup by any machine that runs the `spack_runner`. + +If you have multiple machines that all want to use the same shared Spack +installation you can just have both machines run the `spack_runner`. + +[Spack installation]: https://spack-tutorial.readthedocs.io/en/latest/tutorial_basics.html#installing-spack + +### Managing Spack Python dependencies + +Spack is configured with [SPACK_PYTHON] to ensure that Spack itself uses a +Python virtual environment with a supported copy of Python with the package +`google-cloud-storage` pre-installed. This enables Spack to use mirrors and +[build caches][builds] on Google Cloud Storage. It does not configure Python +packages *inside* Spack virtual environments. If you need to add more Python +dependencies for Spack itself, use the `spack python` command: + +```shell +sudo -i spack python -m pip install package-name +``` + +[SPACK_PYTHON]: https://spack.readthedocs.io/en/latest/getting_started.html#shell-support +[builds]: https://spack.readthedocs.io/en/latest/binary_caches.html + +## Spack Permissions + +### System `spack` user is created - Default + +By default this module will create a `spack` linux user and group with +consistent UID and GID. This user and group will own the Spack installation. To +allow a user to manually add Spack packages to the system Spack installation, +you can add the user to the spack group: + +```sh +sudo usermod -a -G spack +``` + +Log out and back in so the group change will take effect, then `` will +be able to call `spack install `. + +> [!NOTE] +> A background persistent SSH connections may prevent the group change from +> taking effect. + +You can use the `system_user_name`, `system_user_uid`, and `system_user_gid` to +customize the name and ids of the system user. While unlikely, it is possible +that the default `system_user_uid` or `system_user_gid` could conflict with +existing UIDs. + +### Use and existing user + +Alternatively, if `system_user_name` is a user already on the system, then this +existing user will be used for Spack installation. + +#### OS Login User + +If OS Login is enabled (default for most Cluster Toolkit modules) then you can +provide an OS Login user name: + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + settings: + system_user_name: username_company_com +``` + +This will work even if the user has not yet logged onto the machine. When the +specified user does log on to the machine they will be able to call +`spack install` without any further configuration. + +#### Pre-configured user + +You can also use a startup script to configure a user: + +```yaml + - id: spack-setup + source: community/modules/scripts/spack-setup + settings: + system_user_name: special-user + + - id: startup + source: modules/scripts/startup-script + settings: + runners: + - type: shell + destination: "create_user.sh" + content: | + #!/bin/bash + sudo useradd -u 799 special-user + sudo groupadd -g 922 org-group + sudo usermod -g org-group special-user + - $(spack-setup.spack_runner) + + - id: spack-vms + source: modules/compute/vm-instance + use: [network1, startup] + settings: + name_prefix: spack-vm + machine_type: n2d-standard-2 + instance_count: 5 +``` + +### Chaining spack installations + +If there is a need to have a non-root user to install spack packages it is +recommended to create a separate installation for that user and chain Spack installations +([Spack docs](https://spack.readthedocs.io/en/latest/chain.html#chaining-spack-installations)). + +Steps to chain Spack installations: + +1. Get the version of the system Spack: + + ```sh + $ spack --version + + 0.20.0 (e493ab31c6f81a9e415a4b0e0e2263374c61e758) + # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + # Note commit hash and use in next step + ``` + +1. Clone a new spack installation: + + ```sh + git clone -c feature.manyFiles=true https://github.com/spack/spack.git /spack + git -C /spack checkout + ``` + +1. Point the new Spack installation to the system Spack installation. Create a + file at `/spack/etc/spack/upstreams.yaml` with the following + contents: + + ```yaml + upstreams: + spack-instance-1: + install_tree: /sw/spack/opt/spack/ + ``` + +1. Add the following line to your `.bashrc` to make sure the new `spack` is in + your `PATH`. + + ```sh + . /spack/share/spack/setup-env.sh + ``` + +## Deprecations and Breaking Changes + +The old `spack-install` module has been replaced by the `spack-setup` and +`spack-execute` modules. Generally this change strives to allow for a more +flexible definition of a Spack build by using native Spack commands. + +For every deprecated variable from `spack-install` there is documentation on how +to perform the equivalent action using `commands` and `data_files`. The +documentation can be found on the [inputs table](#inputs) below. + +Below is a simple example of the same functionality shown before and after the +breaking changes. + +```yaml + # Before + - id: spack-install + source: community/modules/scripts/spack-install + settings: + install_dir: /sw/spack + compilers: + - gcc@10.3.0 target=x86_64 + packages: + - intel-mpi@2018.4.274%gcc@10.3.0 + +- id: spack-startup + source: modules/scripts/startup-script + settings: + runners: + - $(spack.install_spack_deps_runner) + - $(spack.install_spack_runner) +``` + +```yaml + # After + - id: spack-setup + source: community/modules/scripts/spack-setup + settings: + install_dir: /sw/spack + + - id: spack-execute + source: community/modules/scripts/spack-execute + use: [spack-setup] + settings: + commands: | + spack install gcc@10.3.0 target=x86_64 + spack load gcc@10.3.0 target=x86_64 + spack compiler find --scope site + spack install intel-mpi@2018.4.274%gcc@10.3.0 + +- id: spack-startup + source: modules/scripts/startup-script + settings: + runners: + - $(spack-execute.spack-runner) +``` + +Although the old `spack-install` module will no longer be maintained, it is +still possible to use the old module in a blueprint by referencing an old +version from GitHub. Note the source line in the following example. + +```yaml + - id: spack-install + source: github.com/GoogleCloudPlatform/hpc-toolkit//community/modules/scripts/spack-install?ref=v1.22.1&depth=1 +``` + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [google](#requirement\_google) | >= 4.42 | +| [local](#requirement\_local) | >= 2.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [local](#provider\_local) | >= 2.0.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | +| [local_file.debug_file_shell_install](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [chmod\_mode](#input\_chmod\_mode) | `chmod` to apply to the Spack installation. Adds group write by default. Set to `""` (empty string) to prevent modification.
For usage information see:
https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode | `string` | `"g+w"` | no | +| [configure\_for\_google](#input\_configure\_for\_google) | When true, the spack installation will be configured to pull from Google's Spack binary cache. | `bool` | `true` | no | +| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing startup script. | `string` | n/a | yes | +| [install\_dir](#input\_install\_dir) | Directory to install spack into. | `string` | `"/sw/spack"` | no | +| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | +| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script. Created by this module | `string` | `"/etc/profile.d/spack.sh"` | no | +| [spack\_ref](#input\_spack\_ref) | Git ref to checkout for spack. | `string` | `"v0.20.0"` | no | +| [spack\_url](#input\_spack\_url) | URL to clone the spack repo from. | `string` | `"https://github.com/spack/spack"` | no | +| [spack\_virtualenv\_path](#input\_spack\_virtualenv\_path) | Virtual environment path in which to install Spack Python interpreter and other dependencies | `string` | `"/usr/local/spack-python"` | no | +| [system\_user\_gid](#input\_system\_user\_gid) | GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary. | `number` | `1104762903` | no | +| [system\_user\_name](#input\_system\_user\_name) | Name of system user that will perform installation of Spack. It will be created if it does not exist. | `string` | `"spack"` | no | +| [system\_user\_uid](#input\_system\_user\_uid) | UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary. | `number` | `1104762903` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [controller\_startup\_script](#output\_controller\_startup\_script) | Spack installation script, duplicate for SLURM controller. | +| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for spack, to be reused by spack-execute module. | +| [spack\_path](#output\_spack\_path) | Path to the root of the spack installation | +| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to the Spack profile.d script. | +| [spack\_runner](#output\_spack\_runner) | Runner to be used with startup-script module or passed to spack-execute module.
- installs Spack dependencies
- installs Spack
- generates profile.d script to enable access to Spack
This is safe to run in parallel by multiple machines. Use in place of deprecated `setup_spack_runner`. | +| [startup\_script](#output\_startup\_script) | Spack installation script. | +| [system\_user\_name](#output\_system\_user\_name) | The system user used to install Spack. It can be reused by spack-execute module to install spack packages. | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/main.tf new file mode 100644 index 0000000000..d45f5d1be3 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/main.tf @@ -0,0 +1,120 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "spack-setup", ghpc_role = "scripts" }) +} + +locals { + profile_script = <<-EOF + SPACK_PYTHON=${var.spack_virtualenv_path}/bin/python3 + if [ -f ${var.install_dir}/share/spack/setup-env.sh ]; then + test -t 1 && echo "Running Spack setup, this may take a moment on first login." + . ${var.install_dir}/share/spack/setup-env.sh + fi + EOF + + supported_cache_versions = ["v0.19.0", "v0.20.0"] + cache_version = contains(local.supported_cache_versions, var.spack_ref) ? var.spack_ref : "latest" + add_google_mirror_script = !var.configure_for_google ? "" : <<-EOF + if ! spack mirror list | grep -q google_binary_cache; then + spack mirror add --scope site google_binary_cache gs://spack/${local.cache_version} + spack buildcache keys --install --trust + fi + EOF + + finalize_setup_script = <<-EOF + set -e + . ${var.spack_profile_script_path} + spack config --scope site add 'packages:all:permissions:read:world' + spack config --scope site add 'packages:all:permissions:write:group' + spack gpg init + spack compiler find --scope site + ${local.add_google_mirror_script} + # perform fast install to make sure Spack is fully initialized + spack install xz + spack uninstall --yes-to-all xz + EOF + + script_content = templatefile( + "${path.module}/templates/spack_setup.yml.tftpl", + { + sw_name = "spack" + profile_script = indent(4, yamlencode(local.profile_script)) + install_dir = var.install_dir + git_url = var.spack_url + git_ref = var.spack_ref + chmod_mode = var.chmod_mode + system_user_name = var.system_user_name + system_user_uid = var.system_user_uid + system_user_gid = var.system_user_gid + finalize_setup_script = indent(4, yamlencode(local.finalize_setup_script)) + profile_script_path = var.spack_profile_script_path + } + ) + + install_spack_deps_runner = { + "type" = "ansible-local" + "source" = "${path.module}/scripts/install_spack_deps.yml" + "destination" = "install_spack_deps.yml" + "args" = "-e virtualenv_path=${var.spack_virtualenv_path}" + } + install_spack_runner = { + "type" = "ansible-local" + "content" = local.script_content + "destination" = "install_spack.yml" + } + + bucket_md5 = substr(md5("${var.project_id}.${var.deployment_name}.${local.script_content}"), 0, 8) + # Max bucket name length is 63, so truncate deployment_name if necessary. + # The string "-spack-scripts-" is 15 characters and bucket_md5 is 8 characters, + # leaving 63-15-8=40 chars for deployment_name. Using 39 so it has the same prefix as the + # ramble-setup module's GCS bucket. + bucket_name = "${substr(var.deployment_name, 0, 39)}-spack-scripts-${local.bucket_md5}" + runners = [local.install_spack_deps_runner, local.install_spack_runner] + + combined_runner = { + "type" = "shell" + "content" = module.startup_script.startup_script + "destination" = "spack-install-and-setup.sh" + } +} + +resource "google_storage_bucket" "bucket" { + project = var.project_id + name = local.bucket_name + uniform_bucket_level_access = true + location = var.region + storage_class = "REGIONAL" + labels = local.labels +} + +module "startup_script" { + source = "../../../../modules/scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.runners + gcs_bucket_path = "gs://${google_storage_bucket.bucket.name}" +} + +resource "local_file" "debug_file_shell_install" { + content = local.script_content + filename = "${path.module}/debug_install.yml" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml new file mode 100644 index 0000000000..2ada34471f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - storage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/outputs.tf new file mode 100644 index 0000000000..d94b9757db --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/outputs.tf @@ -0,0 +1,56 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "Spack installation script." + value = module.startup_script.startup_script +} + +output "controller_startup_script" { + description = "Spack installation script, duplicate for SLURM controller." + value = module.startup_script.startup_script +} + +output "spack_path" { + description = "Path to the root of the spack installation" + value = var.install_dir +} + +output "spack_runner" { + description = <<-EOT + Runner to be used with startup-script module or passed to spack-execute module. + - installs Spack dependencies + - installs Spack + - generates profile.d script to enable access to Spack + This is safe to run in parallel by multiple machines. Use in place of deprecated `setup_spack_runner`. + EOT + value = local.combined_runner +} + +output "gcs_bucket_path" { + description = "Bucket containing the startup scripts for spack, to be reused by spack-execute module." + value = "gs://${google_storage_bucket.bucket.name}" +} + +output "spack_profile_script_path" { + description = "Path to the Spack profile.d script." + value = var.spack_profile_script_path +} + +output "system_user_name" { + description = "The system user used to install Spack. It can be reused by spack-execute module to install spack packages." + value = var.system_user_name +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml new file mode 100644 index 0000000000..b7905bbe9e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml @@ -0,0 +1,50 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Create python virtual env for a tool + become: yes + hosts: localhost + vars: + virtualenv_path: ${virtualenv_path} + tasks: + - name: Install dependencies through system package manager + ansible.builtin.package: + name: + - python3 + - python3-pip + - git + register: package + changed_when: package.changed + retries: 5 + delay: 10 + until: package is success + + - name: Create virtualenv for tool + # Python 3.6 is minimum we wish to support due to ease of installation on + # CentOS 7 and Rocky Linux 8. pip 21.3.1 is the *maximum* version of pip + # supported by 3.6. Additionally, recent versions of pip are necessary for + # proper dependency resolution of real-world problems with google-cloud-* + # (and third-party) Python packages (20.3+ probably effective minimum). + ansible.builtin.pip: + name: pip>=21.3.1 + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: /usr/bin/python3 -m venv + + - name: Add google-cloud-storage to virtualenv + ansible.builtin.pip: + name: google-cloud-storage + virtualenv: "{{ virtualenv_path }}" + virtualenv_command: /usr/bin/python3 -m venv diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl new file mode 100644 index 0000000000..ca48a5afa0 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl @@ -0,0 +1,157 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Install Software + hosts: localhost + vars: + sw_name: ${sw_name} + profile_script: ${profile_script} + install_dir: ${install_dir} + git_url: ${git_url} + git_ref: ${git_ref} + chmod_mode: ${chmod_mode} + system_user_name: ${system_user_name} + system_user_uid: ${system_user_uid} + system_user_gid: ${system_user_gid} + finalize_setup_script: ${finalize_setup_script} + profile_script_path: ${profile_script_path} + tasks: + - name: Print software name + ansible.builtin.debug: + msg: "Running installation for software: {{ sw_name }}" + + - name: Add profile script for software + ansible.builtin.copy: + dest: "{{ profile_script_path }}" + mode: '0644' + content: "{{ profile_script }}" + when: profile_script + + - name: Look up user to use for install + block: + + - name: Check if user already exists + ansible.builtin.getent: + database: passwd + key: "{{ system_user_name }}" + + - name: Look up existing user details + ansible.builtin.user: + name: "{{ system_user_name }}" + register: system_user + + rescue: + - name: User did not exist, create group for system user + ansible.builtin.group: + name: "{{ system_user_name }}" + gid: "{{ system_user_gid }}" + system: true + register: system_group + + - name: Create system user + ansible.builtin.user: + name: "{{ system_user_name }}" + comment: "{{ sw_name }} installation" + uid: "{{ system_user_uid }}" + group: "{{ system_group.name }}" + system: true + register: system_user + + - name: Create parent of install directory + ansible.builtin.file: + path: "{{ install_dir | dirname }}" + state: directory + + - name: Set lock dir + ansible.builtin.set_fact: + lock_dir: "{{ install_dir | dirname }}/.install_{{ sw_name }}_lock" + + - name: Acquire lock + ansible.builtin.command: + mkdir "{{ lock_dir }}" + register: lock_out + changed_when: lock_out.rc == 0 + failed_when: false + + - name: Add hostname to lock_dir + ansible.builtin.file: + path: "{{ lock_dir }}/{{ ansible_hostname }}" + state: touch + when: lock_out.rc == 0 + + - name: Clone branch or tag into installation directory + ansible.builtin.command: git clone --branch {{ git_ref }} {{ git_url }} {{ install_dir }} + failed_when: false + register: clone_res + when: lock_out.rc == 0 + + - name: Clone commit hash into installation directory + ansible.builtin.command: "{{ item }}" + with_items: + - git clone {{ git_url }} {{ install_dir }} + - git -C {{ install_dir }} checkout {{ git_ref }} + when: lock_out.rc == 0 and clone_res.rc != 0 + + - name: Transfer ownership to system user + ansible.builtin.file: + path: "{{ install_dir }}" + owner: "{{ system_user.name }}" + group: "{{ system_user.group }}" + recurse: true + follow: false + when: lock_out.rc == 0 + + - name: Finalize setup + ansible.builtin.shell: "{{ finalize_setup_script }}" + when: lock_out.rc == 0 and finalize_setup_script + become: true + become_user: "{{ system_user.name }}" + + - name: Apply chmod + ansible.builtin.file: + path: "{{ install_dir }}" + mode: "{{ chmod_mode | default(omit, true) }}" + recurse: true + follow: false + when: (lock_out.rc == 0) and (chmod_mode != None) + + - name: Release lock + ansible.builtin.file: + path: "{{ lock_dir }}/done" + state: touch + when: lock_out.rc == 0 + + - name: Wait for lock + block: + - name: Wait for lock + ansible.builtin.wait_for: + path: "{{ lock_dir }}/done" + state: present + timeout: 600 + sleep: 10 + when: lock_out.rc != 0 + + rescue: + - name: Timed out on waiting for lock, get lock directory contents + ansible.builtin.find: + paths: "{{ lock_dir }}" + register: lock_dir_contents + + - name: Print lock directory contents, it should contain name of host that is holding lock + ansible.builtin.debug: + msg: "{{ lock_dir_contents.files|map(attribute='path')|map('basename')|list }}" + + - name: Failed to get lock + ansible.builtin.fail: + msg: "Timeout waiting on lock for ${sw_name}, exiting" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/variables.tf new file mode 100644 index 0000000000..85baeec401 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/variables.tf @@ -0,0 +1,106 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created." + type = string +} + +# spack-setup variables + +variable "install_dir" { + description = "Directory to install spack into." + type = string + default = "/sw/spack" +} + +variable "spack_url" { + description = "URL to clone the spack repo from." + type = string + default = "https://github.com/spack/spack" +} + +variable "spack_ref" { + description = "Git ref to checkout for spack." + type = string + default = "v0.20.0" +} + +variable "configure_for_google" { + description = "When true, the spack installation will be configured to pull from Google's Spack binary cache." + type = bool + default = true +} + + +variable "chmod_mode" { + description = <<-EOT + `chmod` to apply to the Spack installation. Adds group write by default. Set to `""` (empty string) to prevent modification. + For usage information see: + https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode + EOT + default = "g+w" + type = string + nullable = false +} + +variable "system_user_name" { + description = "Name of system user that will perform installation of Spack. It will be created if it does not exist." + default = "spack" + type = string + nullable = false +} + +variable "system_user_uid" { + description = "UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary." + default = 1104762903 + type = number + nullable = false +} + +variable "system_user_gid" { + description = "GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary." + default = 1104762903 + type = number + nullable = false +} + +variable "spack_virtualenv_path" { + description = "Virtual environment path in which to install Spack Python interpreter and other dependencies" + default = "/usr/local/spack-python" + type = string +} + +variable "deployment_name" { + description = "Name of deployment, used to name bucket containing startup script." + type = string +} + +variable "region" { + description = "Region to place bucket containing startup script." + type = string +} + +variable "labels" { + description = "Key-value pairs of labels to be added to created resources." + type = map(string) +} + +variable "spack_profile_script_path" { + description = "Path to the Spack profile.d script. Created by this module" + type = string + default = "/etc/profile.d/spack.sh" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/versions.tf new file mode 100644 index 0000000000..ff1180fc1b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/versions.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.0.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/README.md new file mode 100644 index 0000000000..1a3e4cf7e6 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/README.md @@ -0,0 +1,187 @@ +## Description + +Creates a containerised Guacamole instance. Works with the Rocky, Debian and Ubuntu images shown in blueprint example below. + +### Secret Manager Integration + +The VDI module supports flexible Secret Manager integration: + +- **Default Behavior**: Secrets are stored in the deployment project +- **Automatic Password Generation**: If no `secret_name` is provided, random passwords are generated and stored +- **Existing Secret Retrieval**: Provide `secret_name` to use existing secrets from Secret Manager +- **Cross-Project Secrets**: Use `secret_project` to specify a different GCP project if providing secrets + +## Basic Example (Guacamole) + +```yaml +blueprint_name: vdi-test + +vars: + deployment_name: vdi-test + project_id: * project name here * + region: us-central1 + zone: us-central1-a + +deployment_groups: +- group: primary + modules: + - id: network1 + source: modules/network/vpc + settings: + extra_iap_ports: [8080] + firewall_rules: + - name: allow-guacamole-8080-ext + description: Allow external ingress to Guacamole on TCP port 8080 + direction: INGRESS + ranges: ["0.0.0.0/0"] + allow: + - protocol: tcp + ports: ["8080"] + + - id: enable-apis + source: community/modules/project/service-enablement + settings: + gcp_service_list: + - secretmanager.googleapis.com + - storage.googleapis.com + - compute.googleapis.com + + - id: vdi-setup + source: community/modules/scripts/vdi-setup + settings: + vnc_flavor: tigervnc + vdi_tool: guacamole + user_provision: local_users + vdi_user_group: vdiusers + vdi_resolution: 1920x1080 + vdi_users: + # Alice: password generated and saved to Secret Manager in deployment project + - username: alice + port: 5901 + # Bob: existing password is retrieved from Secret Manager in deployment project + - username: bob + port: 5902 + secret_name: a-password-for-bob + # Charlie: password retrieved from Secret Manager in a different project + - username: charlie + port: 5903 + secret_name: charlie-password + secret_project: another-project-id + + - id: guac_vm + source: modules/compute/vm-instance + settings: + instance_image: + # Several supported image families: + family: hpc-rocky-linux-8 + project: cloud-hpc-image-public + #family: debian-11 + #project: debian-cloud + #family: ubuntu-2204-lts + #project: ubuntu-os-cloud + machine_type: e2-highcpu-8 + tags: ["guacamole"] + use: + - network1 + - vdi-setup +``` + +[NVIDIA vWS drivers](https://cloud.google.com/compute/docs/gpus/grid-drivers-table) will be automatically installed on [supported machine types](https://cloud.google.com/compute/docs/gpus#gpu-virtual-workstations). + +Important note: Before deploying the above example you would need to ensure the `secret_name` exists in your local project, and that your service account has sufficient access permissions if retrieving the secret from a separate `secret_project`. The following two example commands show how you can create the two example user's respective secrets: + +```bash +# Create secrets for each user in the deployment project +echo -n "BobPassword123" | gcloud secrets create a-password-for-bob --data-file=- + +# Or create secrets in a different project (if using secret_project) +echo -n "CharliePassword123" | gcloud secrets create charlie-password --data-file=- --project=another-project-id +``` + +### Accessing Guacamole VDI + +After deployment, you can access the VDI in several ways: + +1. **Guacamole Web Interface**: + - Access web interface: + - http://$VM_PUBLIC_IP:8080/guacamole/#/ + - Note: It is not advisable to serve Guacamole directly from a public IP in production environments. You should consider placing the VDI behind a reverse proxy, load balancer, or tunnel to it directly over IAP (see below). + + - Admin credentials: + - Username: `guacadmin` + - Password: Retrieve the `webapp-server...` secret from Secret Manager + +2. **User VDI Access**: + - Each user's credentials are stored in Secret Manager + - Secret names follow the pattern: `vdi-user-password-{username}-{deployment_name}` for auto-generated passwords + - Or use the custom secret name specified in the `vdi_users` configuration + +3. **IAP Tunnel** (for development/testing): + + ```bash + gcloud compute start-iap-tunnel vdi-test-0 8080 \ + --local-host-port=localhost:8080 \ + --zone=us-central1-a + ``` + + - Guacamole will then be accessible from http://localhost:8080/guacamole/ + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [archive](#requirement\_archive) | ~> 2.0 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [archive](#provider\_archive) | ~> 2.0 | +| [google](#provider\_google) | >= 3.83 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | +| [terraform_data.input_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [archive_file.roles_tar](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | n/a | yes | +| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | +| [user\_provision](#input\_user\_provision) | User type to create (local\_users supported. os-login to do. | `string` | `"local_users"` | no | +| [vdi\_resolution](#input\_vdi\_resolution) | Desktop resolution for VNC sessions (e.g. 1920x1080). | `string` | `"1920x1080"` | no | +| [vdi\_tool](#input\_vdi\_tool) | VDI tool to deploy (guacamole currently supported). | `string` | `"guacamole"` | no | +| [vdi\_user\_group](#input\_vdi\_user\_group) | Unix group to create/use for VDI users. | `string` | `"vdiusers"` | no | +| [vdi\_users](#input\_vdi\_users) | List of VDI users to configure. Passwords are handled securely by the Ansible roles: if secret\_name is provided, the password is fetched from Secret Manager; if neither password nor secret\_name is provided, a random password is generated and stored in Secret Manager. If secret\_project is provided, it specifies the GCP project where the secret is stored (defaults to the deployment project). | 
list(object({
    username       = string
    port           = number
    secret_name    = optional(string)
    secret_project = optional(string)
  }))
| `[]` | no | +| [vdi\_webapp\_port](#input\_vdi\_webapp\_port) | Port to serve the Webapp interface from if applicable | `string` | `"8080"` | no | +| [vnc\_flavor](#input\_vnc\_flavor) | The VNC server flavor to use (tigervnc currently supported) | `string` | `"tigervnc"` | no | +| [vnc\_port\_max](#input\_vnc\_port\_max) | Maximum valid VNC port. | `number` | `5999` | no | +| [vnc\_port\_min](#input\_vnc\_port\_min) | Minimum valid VNC port. | `number` | `5901` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [guacamole\_admin\_password\_secret](#output\_guacamole\_admin\_password\_secret) | The name of the Secret Manager secret containing the Guacamole admin password | +| [guacamole\_admin\_username](#output\_guacamole\_admin\_username) | The admin username for Guacamole | +| [startup\_script](#output\_startup\_script) | Combined startup script that installs VDI (VNC, Guacamole, users). | +| [vdi\_runner](#output\_vdi\_runner) | Shell runner wrapping Ansible playbook + roles (for custom-image or direct use). | +| [vdi\_user\_credentials](#output\_vdi\_user\_credentials) | Map of VDI user credentials stored in Secret Manager | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/main.tf new file mode 100644 index 0000000000..38aec83f2c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/main.tf @@ -0,0 +1,134 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "vdi-setup", ghpc_role = "scripts" }) +} + +# Create a tar.gz of roles/ directory +data "archive_file" "roles_tar" { + type = "tar.gz" + source_dir = "${path.module}/roles" + output_path = "${path.module}/roles.tar.gz" +} + +# Generate vars content using templatefile +locals { + vdi_vars_content = templatefile("${path.module}/templates/vars.yaml.tftpl", { + deployment_name = var.deployment_name + project_id = var.project_id + user_provision = var.user_provision + vnc_flavor = var.vnc_flavor + vdi_tool = var.vdi_tool + vdi_user_group = var.vdi_user_group + vdi_webapp_port = var.vdi_webapp_port + vdi_resolution = var.vdi_resolution + vdi_users = var.vdi_users + }) +} + +# Assemble runners +locals { + runners = [ + # Install dependencies + { + type = "shell" + destination = "install-deps.sh" + content = <<-EOT + #!/bin/bash + set -eux + /usr/local/ghpc-venv/bin/python3 -m pip install requests google-auth docker + ansible-galaxy collection install google.cloud + EOT + }, + # Stage roles.tar.gz + { + type = "data" + source = data.archive_file.roles_tar.output_path + destination = "/tmp/vdi/roles.tar.gz" + }, + + # Unpack into /tmp/vdi/roles + { + type = "shell" + destination = "unpack_roles.sh" + content = <<-EOT + #!/bin/bash + set -eux + mkdir -p /tmp/vdi/roles + tar xzf /tmp/vdi/roles.tar.gz -C /tmp/vdi/roles + EOT + }, + + # write out vars file as YAML + { + type = "data" + content = local.vdi_vars_content + destination = "/tmp/vdi/vars.yaml" + }, + + # Run the rendered playbook via ansible-local + { + type = "ansible-local" + content = templatefile("${path.module}/templates/install.yaml.tftpl", + { + roles = ["base_os", "secret_manager", "user_provision", "vnc", "vdi_tool"], + } + ) + destination = "/tmp/vdi/install.yaml" + # Debug mode: + # args = "--extra-vars @/tmp/vdi/vars.yaml -v --extra-vars debug=true" + args = "--extra-vars @/tmp/vdi/vars.yaml" + }, + # Todo: another runner here to delete /tmp/vdi afterwards? + ] + + bucket_name = "${substr(var.deployment_name, 0, 39)}-vdi-scripts-${substr(md5(var.deployment_name), 0, 8)}" +} + +# Bucket to stage runners +resource "google_storage_bucket" "bucket" { + labels = local.labels + project = var.project_id + name = local.bucket_name + location = var.region + uniform_bucket_level_access = true + storage_class = "REGIONAL" +} + +# Use the startup-script module to push and execute them +module "startup_script" { + labels = local.labels + source = "../../../../modules/scripts/startup-script" + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + + runners = local.runners + gcs_bucket_path = "gs://${google_storage_bucket.bucket.name}" + + docker = { + enabled = true + } +} + +# Expose the combined startup script +locals { + combined_runner = { + type = "shell" + content = module.startup_script.startup_script + destination = "install-vdi-and-setup.sh" + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/outputs.tf new file mode 100644 index 0000000000..4e8fa9f21e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/outputs.tf @@ -0,0 +1,44 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "startup_script" { + description = "Combined startup script that installs VDI (VNC, Guacamole, users)." + value = module.startup_script.startup_script +} + +output "vdi_runner" { + description = "Shell runner wrapping Ansible playbook + roles (for custom-image or direct use)." + value = local.combined_runner +} + +output "guacamole_admin_username" { + description = "The admin username for Guacamole" + value = "guacadmin" +} + +output "guacamole_admin_password_secret" { + description = "The name of the Secret Manager secret containing the Guacamole admin password" + value = "webapp-server-password-${var.deployment_name}" +} + +output "vdi_user_credentials" { + description = "Map of VDI user credentials stored in Secret Manager" + value = { + for user in var.vdi_users : user.username => { + username = user.username + port = user.port + secret_name = user.secret_name != null ? user.secret_name : "vdi-user-password-${user.username}-${var.deployment_name}" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles.tar.gz b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..2d0c250eb491e56b53f76f02d13545662f5a9bfa GIT binary patch literal 8732 zcmV+%BIDg3iwFP!00000|LlEhbK5ww=zi9(=#?ug`^MDElAY0(_pV|q8P~};IhLK- zQ%On%ldyy_MQ{O7wyxt-_b1%ncz?+$Jops#vK3`ALwraq5|2is(cNftH?I-L1IlXx z@mpSV@HIhRb&AIBUp80j^?LoV)l&c0>-Fsa&BOL#__u!EXw?r}e}Vc2VVGM2E)e@m z{n=w>C-;Mt9XO%>l#$^`K(pR#!&^#+E{1pSPD(qa9e77<>~ZWs;5nFq7-8u8$R6SF z%|3j@j1%fXvs#B;S)puRmG}N$+JPwzV2q~VQ2_zRAV!444RJBxdmH-#hzB+u`z}GA zjp2@nks8KyQZ4Pkf0-`mwLruJ1UB`jKyQ-u00{-KDkCBMPOWx#cUMIU+A3wknyYK_ z+Pkxp-g&=gHLG>i=E`$1=fLp41H!Na*Hb{g?-CnbyBJ(_2b2LCGK?J%R04d*h#=l@ zA2_`ccZlKA4mgAhMy>;q!d3_#;i(GL1LQ%u+lRA$8D4k$XZ`)s4*Ys{`Sbg$OZc^W zanU`$JnQx0{RN!7KR-RYJbQoMhxc!wd;VYWA7|&M`+$iUVFvi#XA*wOfJi)IMF2%-Vl7u#)Qk&TXz^URO@gNh#h{Pf<0U(c)YZq6q1L6weRXG-e8)S=4Nv_#2AOj!Sw`hoY z2R>EqZ*1JE>_g?AV>ZFex~FVt*_2`V(76t{b>C=IzNp3%&mQ=*>G@4zatS%aNGvQaxMJ zCC6*G@f>mkrj10)z`>Ig^88zgD({92XE5mCiG1dg>-$C{`3v8R#LuBS@tp9tTrtPO zoMTTAQ$>@4?z#N% zkRl3r37^b5e}R{?^M=RAvTDu+61XEFk?@uP7#563#5C$900+Bh+JS@FIox9AVHZXi zIhb*^V_4v+oncyFNwgDp@^9mqfu9g;tbpcDT!px#4AyM)s zYSZx;c}_f%J$B~74VLBHjyYHsC$5$wrVYbLNNG|FIICR~FKo-le=G9;LUIy#Cl5q(lz@zk{1~{m$b++!Q;@2G z6qTty^5tdf6c(SvCaN<_ca#$4sbiNc>7$p{70mIgP+7FfvcftPddrVdmdO9$I`BmB zZO{Mh`e8dO|J$vjLjG@~6!O22|67s&SBjJTna=;I8Ggi!+)Ux^$q7u%e!0dsl;IFf zM2lO6>N%6iR@lo|Wv&LR5veaj!Ublk-oVrv*duJ;ni}>9c|+_B?nc<_ zzzuRaPQ!YNuY=+9Ky_pnF?z;)*n#qDJ$B&XVHWl(7Y+@CTGv(8l#t}UmxhvE5dCUk zCOcr52X5Ha4I=KMH93(;0CE$~GY4_N1KY;f!OlGT5lF<-^^q8@1veiPLq|E(V!%ql zlbYZv6cYWCy_eHnDn3U9sC_3%Bo~`iW~a8k>L{L??1BrPY#+@^p{J@V%=U4XLbRgA z1yCf2bzv`e~t@6@T*EVI;9=8B)n6r*)9#7>QGUcl)W{%_JP1PhuVq(@tvU` zSQElTfL!a2vRiHdSgV)XkP*kI@Z1t|CG}}F^7?s;1-w1^0Apm2h=)N;eGE5@j^%+7 zIY=OgJg?@SKh@dOkXzU+go}6rGY`2TaKD)MwMnC<6cf+JH8Z*;M^`;&PpigzC=~Ny zC)OL~g4Qt`wxksn5ioCHQ%_(|bYQC7?vf`$e_+NNsPQ6P^zH>iwg5h&Or$Wf+;1<_ zsd3!OJgpj_s@!*p*satmd!MYv7p>f8p$NynL1W@h&6=zOQ#i8jU=BnE605TJsh$BO zI&GzBgK*%1?^7l>aN5|h6e7Y0Mx~nE@j=v$1}zP)vK-+Kx9kfLnfW?Gn2t`Alu{_QO@RA!{eEZ=l#px2>gu0nFKg`yf zMJdrO21*I*z#=Pr$tdXN_z?ON$_3%oB%sPGRfAA5mFKKSX$JDd{mv%9kDUnCgI{1KoFCT%T!XL-^6#zssglS*Pgt)srBmFWpK{3`jxU(*hNh!3L`>L;l>~G5_e)R0Of9FEAbhCY6 zbV3xPmldC=Q>9mIc9T_^Ef=f)`%sSl1SPHiZN%MorY!USZ69Uhe-Dn1nuY$ijZ)}; zh5on6_}^Tu;^{%a|NPgDjVt?=e@Z@O-G75<-%W-1ZY{1i5!OrD(7JClOh=Z3C)iEB z2n2DhF|rxu*|1)misHTU9VDch<5NxqWmB-#CjVQgi9qi|?_J9C7pUh>R}6&3K6WjJ zU5v~Xd5-6Gpg~i~i5U)>3@Uh+uHl5(3w3G6?cCf&< zvS1<4$#`YP_s_;$Bbg`?H!A{FYeMDzw_+Xzl3g#exoeHfx7+I9WLb|AqP*6tm$kPBIEg( zbOUI7k?dn`RlRR3jYcKyEug(O4U&_~hxMfLF`aPUOgcYXP`*HSIlU?G&H1p!4xAR0 z4K}e8%d~7yd8kTHOHz&vwRVbKtgN#hhgDf{b5%u!rQxNs?<$8$n{bwR72J=aXikqhq6~na*QOSLv?svJ=tteyI7VdO0y7X{X zste0<2=`qyO)!ph;AgV>Q=>mqL+Y^Cb?`T*izvQx7Q_r1HHa%q+~0vU^~Wn5+YF1r z7!f(%Zn_t%Of2`%tMl zcvABM*98}QLoqr&Xv$L$PJ?=fuaWNqa>m449bDaN zpjv%%kWX}YDg2rTDvS^xNvh>&g603F*4)t61rw27=}F|yPoCuwAMIB72sPV>l|A?h zf$w8Bd#`|OTT-l`w^(0g^)==EqU8{DKb$R%x-CB^>{d>n*w{k0O#@Hx@Z?idAeA>A zi7QxuQ&$`s0RuH!mI(U3Et@;1f|DQR>8kW%YEF=Nz;Q<`%yL391XLC87$O0Fl^TYdoEBgkbQgS+DOytNzm$fF1ci90PJ0M$XhEA4-vd z*+&N7I-68X2VKVr*F4%|p#?5BiMxAu&-xo}%&mRKsp zA2725u41T`XvBWHrqa6sdyLp4aEZz_mJI^rDw;~IoO%=nRk@V<8vcnx2C`b{G9}+A zWrR_a`At2UAl3mQ@VJ!n;nSmIC+=ntUZsTYd`tDlnWh=Qq8>GX+SEV$mZ%#>;{9I2 z<^OOc*Zz}FVNV`ong91;Jsbajbkr>Tzqe8f`%hv2d9M8@YlkpL=}@~+3In+Tk9rt< zMkhq4s7(uuMCA^YCk|0kF{Ok^NmZ3kjS4bf6;0-9QLQ35PPih(P^cuTq@YdLO1QQO zi6Jkcsc{-GTVq#=@foB+iK9@V{TkBZZhRlLmsE2z?F@jS(=)HR zNMJ5HS`l5zDzs%acd4O?kK;tsIr%F>)hk1S@D&6jV>mvBvc^q0ts{OsJ=5zOuQ;RQ z3^}Z#B`RGbZ6S?7*61;MdR1lQQ+l|A6MA?){0YF7MjiL54|B3YvtFNX_kAiQuQI`v zOu&u9T==X!V+BdrKjvW_b|Z+xe)8fq zz7tQ0ttQVDVn5)K`kUEP)}o}ope)ujVqzy=R$TocQtBCm^oKUnqOyARn{vQ#FXt#2O(30Om&+NR@i;1Myc z>h*A|TA?3s!H74k?1%dGt9@+>LGH0S3hnLZjb;4o2VO9~#;npYGKVs`uZSnWg7!~| zPd|m_aUUL1*aC=ueKCJMK7MzrwqD;bwVSd3y3|IlrmIc30WR_XJ7^tbzyEi5*lZQ{ z-))q_{#)38H)a2gGem~gAr8BCzy%#k*k)BGz86QhDwentYTxhQV|^rYz-u>}mAw?yCc#$R5%r8W0_*_Cf`fkt zoZ!~uFg5G(Ut+b|dmNfAx$?%Ji^?D^8HW%3sLk#(duqD%vbrqXUeL<_# z#~$aS!7ZN3=4+y&Mw|iTsQfJ_ekdodA0=$e8ss!Wx{d?-a}5d|g36z{o29AyJ-TGo(OHDL13MKR*vTR3l_Vy@kLE$6(Eqkk3jMFp z|2C!n84aMK6rhS0Ea`mwTGo_eJxf(B<*?q6BWF90^gG>qWWviziJa?@|L_0P7iJ-clvpA+z_E*MsTR%ZzN($pewXJ*&z`jNz4$WDa!bAWLMv68xkt8` z<722K2CjrNe&2_8bV$O)_=~3*RbFwXln+$E!gi+_j#~LM$nvL@T=}0`tLKlgME*A# z^_lN~HX6nEKeti}`CrKYXUhNYO~OTaMv=l-Q0j&sfDVw2#?-}|%mKc{|GU*_X6^s2 zqoa1g|F=;J{$KF_4WIu$P%efH_y6;qx>;A>kRT$m%M#4>u`TD!5Zix^!P8jvvPZSF zjn|kt8KwH%Xe^r1@5pG|ZdbyShcQ*ws4rrsXF1n33G&>Raw5)k_B+9+)V@^#Tq>kI z&XhKL#jh&p50rIg@RsR@@Gv*^{H~G*Tt_ueVo7o~VVvBtCu!N4{U)?-HV9!ez(=tC ziimcGGSE2)k9Az)H3sEIlLOryB4ThuS#l|B6U1sP@M?O%DyJqTogiaO1JQwlgg5Rb z<{})@rk;mwK_*1x4QA8v7aq8`62_WzUaZ{C}&ygk8O7R4L zs4ez@?*;aFn2_7hM;?&AF?M^52ijXA}# z;1_g@!68G;#qQodjK;{8|6^fS0ZkBbO?00uCl|f$WeCjKEvre8Oz1sS7Z?x$(8N`)_5$I56%iww!$>=yGhy`7vzFOz#M zCohPD9bzl)X5jX)xTEZL1;9;Bv!m3o0}qSzw~lq4Ff~OcPAt6CPgRnN(O-|D9=&b9n$kdT-? zfaOEGbC!bmbo>2Z-(Q>_=cbD=UJS_wr>}E5 z%wyNe9a*W%F1fD7EH!a7S;XZvE-dT`HuWj-M7Rx&zm&zWIv4{r=Q9(EEN z>F4+T%k%Cpz2o$Cf zj&cgb=u#m$k>zOlVi^rSI+E+7K5;RJK-;m^*BE4{#^|?#p>?jQGht3d*Tyr)e7s*J z$dDt&AxAoprnnLOaZ2X?Z^NFSx-9emXt%T9e>gZgD*QjUQi}V(;{I<__kZQ+`);8i zDbHpvOnVrlQvXRll=9w->Q-_FIj6rMdqljvK|DIv?^YNjs+%{+uaw<#pNgC@?#!zd zh6toCp#oArYt|Mzx^Wx>bu6F7<1JdDau2INbw zRMNKf0rmtDQ|R_Vr+w=8-t|r{K|dZ4ZIOR-@&1?iq;J}&H^2VeyXdJ?HXK7GJx&F> z=cgd1zIu1n?_E?%04gt&hwl=OWAW%`=l$NrC7hjKzE6yosJWY2l_Q&*>B z`q+JU)$7A<7LKdkj*6u9v;G*gMl&e4~Rc$ zRgvQ;iSzhaiPC26R(#5YX=Sc?l5a!oni@%DC;LdG#C>>3xz?$gs^XqRvPSprocZo$ z7bhXv)>xsa73@Y7zCVNVs8(`e5*Jct161QK%**szRMY*2w^o>qc{LuR(}_W)*D9Z1 z{Bj_7PrTNN!S}NZZ@QOQk2b+YuX|eA+uyBReK_r2_SB!Jy>~tHXZQ4%v-7k5W$&W0 zr`JHjwws>+Uda$<EJh+mQaR_NC$O3IVEkRM8VT~ZU^mU? z+H5AG{ff`X1PN^2;%Oz`61_^CrMoQ8D&_q!`A!-BJMQ$qp`6ORlTP>t@HO;D+%13p zEc^KU`Sa&RwJPO(fO4g>uqP&aT3Ex=V|nF0Id4;O6TEX>vfC>El)!D3hboHR1|R)) z`d^OzpBsAdoH3T@|MkPe?EUXiqgB}dw^A%CbTf{-8|qidZ%85q*!uLL5 zSOSn>QkkL7uim{2+p0LLigAHTDzf`f>Hpld)Ms*aWslzNnm?5rP2$f|Bu({D8)OI* z&;FLKAF3>~|F#;f%>Hj49TfK8t&~z>04@x`8+QP?Bty(To}c8nO@w*BGj0>J8N-i{ z6G_7J#~6vZW2ijitB#?QlK&|U*)ji5EzCy8SZ4oiw6pm?4h|2R1^?ehDcFC({-4YK z6Q4V_sdqz$0aGm!ABNOV6Y=RE&rg73A(6;u8MtKC>KLl~+f2^Du#E)xtxK?m5f46@)}PF({S*(0^XD%>y(33<3JvpHG9RuO(@ z2{r{n_^H`NR8}?z{8aLsoCG`HmT=}yFrt~o6YDG_`t|3RPY}x|^u3jxqRf<({vRKj zHaf;K`_HWZ-@#$C$p5#MQt1DM{$Ke2{dr5w|MSjVn;ByX|35ft9A>}&(Ksx=|F)G< z@c)AUZ#e&}Jej2IK#9+R&HY}-Oy<|$x}CfVemE6lwX4y^&5$@IDNSCyMD`r2IJaV((U>-W#zpAUZN zo_F8&E>iDKUtjeHVFmej_w@9l*AE@F@IAKmdGP+m#;tFHWu2b)2}kLirHSs^6{lIS zKFcyKVtYi>Q_Ron2w!31+3JY{)(!k)rDfFh%lXL<*X!UK+jOj5AQr7jgsV5?9t^k& zM~S_^nf;l7ZR50ZnA(w2gr!Z(n*Xev=Tz!D8GGdN>FIXgW_npOrmW8}r*2@rqHX$! zpFPpQqV&IH=!ZtdMs_+Qb;k_J5<9_5W(O>V^N;R?5Q*r5x#R{(8|hBD{Dkb&Ol4PKd3)a}SS?$Km6o z67}!`u;<90UKEVtM=Epn|LncuGsal9|7ZO_j~dORV*hWWe7f?8_)+-Ag=qW(C( zpVrr{5r23=O5GcQ*)gATmkz4}u^|>=if8k)RQlA{r1Pb8IrZ`J4Q02)8|^`5(Jqxf8Lz=FrC*UJu=9F)JPuqz)QKU4{Cls(_b5yEf8?$4^f8w0|90{$ zp!fel%l`xb0RR7tShsZm G6afIrxTI- + {%- if vnc_flavor|lower == 'tigervnc' -%} + tigervnc-standalone-server + {%- elif vnc_flavor|lower == 'tightvnc' -%} + tightvnc-standalone-server + {%- else -%} + tightvnc-standalone-server + {%- endif %} + xorg_dev_packages: ['pkg-config', 'xserver-xorg-dev', 'libx11-dev', 'libxext-dev', 'libglvnd-dev', 'build-essential'] + +- name: Install desktop prerequisites & VNC package (Debian) + ansible.builtin.package: + name: "{{ gui_packages + [vnc_pkg] + xorg_dev_packages }}" + state: present + retries: 10 + delay: 3 + +- name: Install kernel headers for running kernel (Debian) + ansible.builtin.package: + name: "linux-headers-{{ ansible_kernel }}" + state: present + retries: 10 + delay: 3 diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml new file mode 100644 index 0000000000..f148682825 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml @@ -0,0 +1,69 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NVIDIA vWS (GRID) driver setup. See: +# https://cloud.google.com/compute/docs/gpus#gpu-virtual-workstations +# https://cloud.google.com/compute/docs/gpus/grid-drivers-table + +--- + +- name: Get GCP machine type from metadata server + ansible.builtin.uri: + url: http://metadata.google.internal/computeMetadata/v1/instance/machine-type + method: GET + headers: + Metadata-Flavor: Google + return_content: yes + register: gcp_machine_type + +- name: Extract short machine type + ansible.builtin.set_fact: + gcp_machine_type: "{{ gcp_machine_type.content.split('/')[-1] }}" + +- name: Extract machine family from machine type + ansible.builtin.set_fact: + machine_family: "{{ gcp_machine_type.split('-')[0] }}" + +- name: Check if machine type is supported for NVIDIA vWS + ansible.builtin.set_fact: + is_nvidia_vws_supported: "{{ machine_family in ['g2', 'n1'] }}" + +# Additional drivers for other machine type families can go here +- name: Set driver download URL based on machine type + ansible.builtin.set_fact: + driver_url: >- + {%- if is_nvidia_vws_supported -%} + https://storage.googleapis.com/nvidia-drivers-us-public/GRID/vGPU18.1/NVIDIA-Linux-x86_64-570.133.20-grid.run + {%- else -%} + + {%- endif %} + +# Include distribution-specific GPU driver setup +- name: Include Ubuntu GPU driver setup + ansible.builtin.include_tasks: gpu_driver_setup/ubuntu.yaml + when: + - ansible_facts.distribution == 'Ubuntu' + - is_nvidia_vws_supported + +- name: Include Rocky Linux GPU driver setup + ansible.builtin.include_tasks: gpu_driver_setup/rocky.yaml + when: + - ansible_facts.distribution == 'Rocky' + - is_nvidia_vws_supported + +- name: Include Debian GPU driver setup + ansible.builtin.include_tasks: gpu_driver_setup/debian.yaml + when: + - ansible_facts.distribution == 'Debian' + - is_nvidia_vws_supported diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/debian.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/debian.yaml new file mode 100644 index 0000000000..7bd4f3699c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/debian.yaml @@ -0,0 +1,30 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# Block for NVIDIA vWS driver setup (Debian) +- block: + - name: Download NVIDIA vWS driver + ansible.builtin.uri: + url: "{{ driver_url }}" + method: GET + dest: /tmp/nvidia-driver.run + mode: '0755' + status_code: + - 200 + - 304 + - name: Install NVIDIA vWS driver (Debian) + ansible.builtin.command: + cmd: /tmp/nvidia-driver.run --dkms --silent + creates: /usr/bin/nvidia-smi diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/rocky.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/rocky.yaml new file mode 100644 index 0000000000..260ca90d19 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/rocky.yaml @@ -0,0 +1,36 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# Block for NVIDIA vWS driver setup (Rocky Linux) +- block: + - name: Install NVIDIA Vulkan ICD loader (Rocky Linux) + ansible.builtin.package: + name: vulkan-loader + state: present + retries: 10 + delay: 3 + - name: Download NVIDIA vWS driver + ansible.builtin.uri: + url: "{{ driver_url }}" + method: GET + dest: /tmp/nvidia-driver.run + mode: '0755' + status_code: + - 200 + - 304 + - name: Install NVIDIA vWS driver (Rocky Linux) + ansible.builtin.command: + cmd: /tmp/nvidia-driver.run --dkms --silent + creates: /usr/bin/nvidia-smi diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/ubuntu.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/ubuntu.yaml new file mode 100644 index 0000000000..8cef6585a0 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/ubuntu.yaml @@ -0,0 +1,73 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# Block for NVIDIA vWS driver setup (Ubuntu) +- block: + - name: Download NVIDIA vWS driver + ansible.builtin.uri: + url: "{{ driver_url }}" + method: GET + dest: /tmp/nvidia-driver.run + mode: '0755' + status_code: + - 200 + - 304 + + - name: Verify GCC version before driver installation + ansible.builtin.shell: gcc --version | head -1 + register: gcc_version_check + changed_when: false + + - name: Debug GCC version before driver installation + ansible.builtin.debug: + msg: "GCC version before driver installation: {{ gcc_version_check.stdout }}" + + - name: Install NVIDIA vWS driver (Ubuntu) + ansible.builtin.command: + cmd: /tmp/nvidia-driver.run --dkms --silent + creates: /usr/bin/nvidia-smi + register: nvidia_install_result + failed_when: false + + - name: Check if NVIDIA driver installation succeeded + ansible.builtin.stat: + path: /usr/bin/nvidia-smi + register: nvidia_smi_check + + - name: Debug NVIDIA installation result + ansible.builtin.debug: + msg: + - "NVIDIA installer exit code: {{ nvidia_install_result.rc }}" + - "nvidia-smi exists: {{ nvidia_smi_check.stat.exists }}" + - "Installation stdout: {{ nvidia_install_result.stdout }}" + - "Installation stderr: {{ nvidia_install_result.stderr }}" + + - name: Fail if NVIDIA driver installation failed + ansible.builtin.fail: + msg: "NVIDIA driver installation failed. Check /var/log/nvidia-installer.log for details." + when: + - nvidia_install_result.rc != 0 + - not nvidia_smi_check.stat.exists + + - name: Verify NVIDIA driver installation + ansible.builtin.command: nvidia-smi + register: nvidia_smi_test + changed_when: false + when: nvidia_smi_check.stat.exists + + - name: Debug NVIDIA driver verification + ansible.builtin.debug: + msg: "NVIDIA driver verification: {{ nvidia_smi_test.stdout }}" + when: nvidia_smi_check.stat.exists diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml new file mode 100644 index 0000000000..209155873b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml @@ -0,0 +1,56 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Gather distribution facts + ansible.builtin.setup: + gather_subset: platform + +# GPU / vWS driver setup +- name: Check if NVIDIA GPU is present + ansible.builtin.shell: lspci | grep -i nvidia + register: nvidia_gpu_present + failed_when: false + +# Include distribution-specific tasks +- name: Include Ubuntu-specific tasks + ansible.builtin.include_tasks: ubuntu.yaml + when: ansible_facts.distribution == 'Ubuntu' + +- name: Include Rocky Linux specific tasks + ansible.builtin.include_tasks: rocky.yaml + when: ansible_facts.distribution == 'Rocky' + +- name: Include Debian-specific tasks + ansible.builtin.include_tasks: debian.yaml + when: ansible_facts.distribution == 'Debian' + +# Common tasks for all distributions +- name: Ensure requests package is installed + pip: + name: requests + state: present + +- name: Ensure Python & tooling are installed + ansible.builtin.package: + name: + - python3 + - python3-pip + - jq + state: present + +# GPU / vWS driver setup +- name: Provision GPU setup + ansible.builtin.include_tasks: gpu_driver_setup.yaml + when: nvidia_gpu_present.rc == 0 diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/rocky.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/rocky.yaml new file mode 100644 index 0000000000..132e53fa89 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/rocky.yaml @@ -0,0 +1,42 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Set Rocky Linux specific variables + ansible.builtin.set_fact: + gui_packages: ['@Xfce','@base-x'] + vnc_pkg: >- + {%- if vnc_flavor|lower == 'tigervnc' -%} + tigervnc-server + {%- elif vnc_flavor|lower == 'tightvnc' -%} + tightvnc-server + {%- else -%} + tightvnc-standalone-server + {%- endif %} + xorg_dev_packages: ['pkgconfig', 'xorg-x11-server-devel', 'xorg-x11-util-macros', 'libglvnd-devel'] + +- name: Update repository cache & install EPEL (Rocky Linux only) + ansible.builtin.package: + name: epel-release + state: present + update_cache: yes + retries: 10 + delay: 3 + +- name: Install desktop prerequisites & VNC package (Rocky Linux) + ansible.builtin.package: + name: "{{ gui_packages + [vnc_pkg] + xorg_dev_packages }}" + state: present + retries: 10 + delay: 3 diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml new file mode 100644 index 0000000000..e467a0cca1 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml @@ -0,0 +1,138 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Set Ubuntu specific variables + ansible.builtin.set_fact: + gui_packages: ['xfce4', 'xserver-xorg-core'] + vnc_pkg: >- + {%- if vnc_flavor|lower == 'tigervnc' -%} + tigervnc-standalone-server + {%- elif vnc_flavor|lower == 'tightvnc' -%} + tightvnc-standalone-server + {%- else -%} + tightvnc-standalone-server + {%- endif %} + xorg_dev_packages: ['pkg-config', 'xserver-xorg-dev', 'libx11-dev', 'libxext-dev', 'libglvnd-dev'] + +- name: Install desktop prerequisites & VNC package (Ubuntu) + ansible.builtin.package: + name: "{{ gui_packages + [vnc_pkg] + xorg_dev_packages }}" + state: present + retries: 10 + delay: 3 + +- name: Install kernel headers for running kernel (Ubuntu) + ansible.builtin.package: + name: "linux-headers-{{ ansible_kernel }}" + state: present + retries: 10 + delay: 3 + +# GCC version detection and installation for NVIDIA driver compatibility +- name: Get kernel build GCC version + ansible.builtin.shell: cat /proc/version | grep -o 'gcc-[0-9]*' | head -1 + register: kernel_gcc_version + changed_when: false + +- name: Extract GCC major version from kernel + ansible.builtin.set_fact: + required_gcc_version: "{{ kernel_gcc_version.stdout.split('-')[1] if kernel_gcc_version.stdout else '11' }}" + +- name: Debug kernel GCC version extraction + ansible.builtin.debug: + msg: + - "Raw kernel GCC version: {{ kernel_gcc_version.stdout }}" + - "Extracted required GCC version: {{ required_gcc_version }}" + +- name: Get current GCC version + ansible.builtin.shell: gcc --version | head -1 | grep -o '[0-9]*\.[0-9]*' | head -1 + register: current_gcc_version + changed_when: false + +- name: Extract current GCC major version + ansible.builtin.set_fact: + current_gcc_major: "{{ current_gcc_version.stdout.split('.')[0] if current_gcc_version.stdout else '11' }}" + +- name: Debug current GCC version extraction + ansible.builtin.debug: + msg: + - "Raw current GCC version: {{ current_gcc_version.stdout }}" + - "Extracted current GCC major: {{ current_gcc_major }}" + +- name: Debug GCC version information + ansible.builtin.debug: + msg: + - "Kernel was built with GCC {{ required_gcc_version }}" + - "Current system GCC version: {{ current_gcc_major }}" + - "GCC update needed: {{ required_gcc_version != current_gcc_major }}" + +- name: Install required GCC version if different from current + block: + - name: Install GCC {{ required_gcc_version }} + ansible.builtin.package: + name: "gcc-{{ required_gcc_version }}" + state: present + retries: 10 + delay: 3 + + - name: Install G++ {{ required_gcc_version }} + ansible.builtin.package: + name: "g++-{{ required_gcc_version }}" + state: present + retries: 10 + delay: 3 + + - name: Set GCC {{ required_gcc_version }} as default + ansible.builtin.alternatives: + name: gcc + path: "/usr/bin/gcc-{{ required_gcc_version }}" + priority: "{{ required_gcc_version }}" + link: /usr/bin/gcc + + - name: Set G++ {{ required_gcc_version }} as default + ansible.builtin.alternatives: + name: g++ + path: "/usr/bin/g++-{{ required_gcc_version }}" + priority: "{{ required_gcc_version }}" + link: /usr/bin/g++ + + - name: Verify GCC version after update + ansible.builtin.shell: gcc --version | head -1 + register: updated_gcc_version + changed_when: false + + - name: Debug updated GCC version + ansible.builtin.debug: + msg: "Updated GCC version: {{ updated_gcc_version.stdout }}" + + - name: Verify G++ version after update + ansible.builtin.shell: g++ --version | head -1 + register: updated_gpp_version + changed_when: false + + - name: Debug updated G++ version + ansible.builtin.debug: + msg: "Updated G++ version: {{ updated_gpp_version.stdout }}" + + - name: Verify alternatives are set correctly + ansible.builtin.shell: update-alternatives --display gcc + register: gcc_alternatives_check + changed_when: false + + - name: Debug GCC alternatives status + ansible.builtin.debug: + msg: "GCC alternatives status: {{ gcc_alternatives_check.stdout }}" + + when: required_gcc_version != current_gcc_major diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/defaults/main.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/defaults/main.yaml new file mode 100644 index 0000000000..15a621b3ad --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/defaults/main.yaml @@ -0,0 +1,16 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +secret_project: "{{ project_id }}" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml new file mode 100644 index 0000000000..50eab08ffa --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml @@ -0,0 +1,102 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# General secret handling +- name: Generate random database password + ansible.builtin.set_fact: + database_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}" + +- name: Generate random webapp admin password + ansible.builtin.set_fact: + webapp_admin_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}" + +- name: Compute Webapp admin password hash to save to Webapp server + ansible.builtin.set_fact: + webapp_admin_hash: "{{ webapp_admin_password | hash('sha256') | upper }}" + +- name: Get access token for Secret Manager API + ansible.builtin.uri: + url: "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" + method: GET + headers: + Metadata-Flavor: Google + return_content: true + register: access_token_result + +- name: Create a GCP secret for webapp server login + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets?secretId=webapp-server-password-{{ deployment_name }}" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + replication: + automatic: {} + status_code: [200, 409] # 409 = already exists + register: webapp_secret_create + +- name: Add webapp password version to secret + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/webapp-server-password-{{ deployment_name }}:addVersion" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + payload: + data: "{{ webapp_admin_password | b64encode }}" + register: webapp_secret + +- name: Create a GCP secret for database password + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets?secretId=db-password-{{ deployment_name }}" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + replication: + automatic: {} + status_code: [200, 409] # 409 = already exists + register: database_secret_create + +- name: Add database password version to secret + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/db-password-{{ deployment_name }}:addVersion" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + payload: + data: "{{ database_password | b64encode }}" + register: database_secret + +# Per-user secret handling +- name: Generate per-user secrets and enrich list + include_tasks: user_secret_tasks.yaml + loop: "{{ vdi_users }}" + loop_control: + loop_var: item + +- name: Set enriched vdi_users_updated variable + ansible.builtin.set_fact: + vdi_users_updated: "{{ vdi_users_updated }}" + when: vdi_users_updated is defined diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml new file mode 100644 index 0000000000..00e7fe540d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml @@ -0,0 +1,127 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +- name: Compute default secret_name if none provided + set_fact: + secret_name: "vdi-user-password-{{ item.username }}-{{ deployment_name }}" + when: + - item.password is not defined + - item.secret_name is not defined + +- name: Determine password source for user {{ item.username }} + ansible.builtin.set_fact: + password_source: >- + {%- if item.secret_name is defined -%}secret + {%- elif item.password is defined -%}password + {%- else -%}generate{%- endif -%} + +- name: Fetch existing Secret Manager password for user {{ item.username }} + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets/{{ item.secret_name }}/versions/latest:access" + method: GET + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + return_content: true + register: sm_result + failed_when: false + when: password_source | trim == "secret" + +- name: Set VDI user password from Secret Manager + ansible.builtin.set_fact: + vdiuser_password: "{{ sm_result.json.payload.data | b64decode }}" + when: password_source == "secret" and sm_result.status == 200 + +- name: Generate random password for user {{ item.username }} + ansible.builtin.set_fact: + vdiuser_password: "{{ lookup('password', '/dev/null length=16 chars=ascii_letters,digits') }}" + when: password_source == "generate" + +- name: Set user password from provided value + ansible.builtin.set_fact: + vdiuser_password: "{{ item.password }}" + when: password_source == "password" + +- name: Create secret for user {{ item.username }} (provided password) + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets?secretId=vdi-user-password-{{ item.username }}-{{ deployment_name }}" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + replication: + automatic: {} + status_code: [200, 409] # 409 = already exists + when: password_source == "password" + +- name: Store provided password in Secret Manager for user {{ item.username }} + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets/vdi-user-password-{{ item.username }}-{{ deployment_name }}:addVersion" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + payload: + data: "{{ item.password | b64encode }}" + when: password_source == "password" + +- name: Create secret for user {{ item.username }} + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets?secretId={{ item.secret_name | default(secret_name) }}" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + replication: + automatic: {} + status_code: [200, 409] # 409 = already exists + when: password_source == "generate" + +- name: Store generated password in Secret Manager for user {{ item.username }} + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets/{{ item.secret_name | default(secret_name) }}:addVersion" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + payload: + data: "{{ vdiuser_password | b64encode }}" + when: password_source == "generate" + +- name: Build enriched user list + ansible.builtin.set_fact: + vdi_users_updated: >- + {{ + (vdi_users_updated | default([])) + + [ + item + | combine({ 'password': vdiuser_password }) + | combine( + (vnc_flavor | lower in ['tightvnc', 'tigervnc']) + | ternary({ + 'vncserver_password': lookup('community.general.random_string', length=8, special=false), + 'display_number': item.port | int - 5900 + }, {}) + ) + ] + }} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml new file mode 100644 index 0000000000..39ddf37c18 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml @@ -0,0 +1,51 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +- name: Provision VDI users (custom or generated passwords) + block: + + # Create the Linux user with a hashed password + - name: Create VDI user {{ item.username }} + ansible.builtin.user: + name: "{{ item.username }}" + groups: "{{ vdi_user_group }},wheel" + shell: /bin/bash + create_home: yes + append: true + password: "{{ item.password | password_hash('sha512') }}" + update_password: on_create # sets pw if user is newly created + + # SSH key setup + - name: Ensure .ssh directory for {{ item.username }} + ansible.builtin.file: + path: "/home/{{ item.username }}/.ssh" + state: directory + owner: "{{ item.username }}" + group: "{{ vdi_user_group }}" + mode: '0700' + + - name: Generate SSH key pair for {{ item.username }} + community.crypto.openssh_keypair: + path: "/home/{{ item.username }}/.ssh/id_rsa" + owner: "{{ item.username }}" + group: "{{ vdi_user_group }}" + mode: '0600' + register: ssh_key + + - name: Add public key to authorized_keys for {{ item.username }} + ansible.builtin.authorized_key: + user: "{{ item.username }}" + key: "{{ ssh_key.public_key }}" + state: present diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml new file mode 100644 index 0000000000..569db3dd6c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml @@ -0,0 +1,40 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +- name: Ensure 'wheel' group is present + group: + name: wheel + state: present + +- name: Ensure VDI user group exists + ansible.builtin.group: + name: "{{ vdi_user_group }}" + state: present + +# Run localā€user provisioning if selected +- name: Provision local users + ansible.builtin.include_tasks: local_users.yaml + loop: "{{ vdi_users_updated }}" + loop_control: + loop_var: item + when: user_provision | lower == 'local_users' + +# Run OS Login tasks if selected +- name: Provision OS Login users + ansible.builtin.include_tasks: os_login.yaml + loop: "{{ vdi_users_updated }}" + loop_control: + loop_var: item + when: user_provision | lower == 'os_login' diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/os_login.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/os_login.yaml new file mode 100644 index 0000000000..3039c31d62 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/os_login.yaml @@ -0,0 +1,14 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml new file mode 100644 index 0000000000..4bb770a0c1 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml @@ -0,0 +1,212 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +- name: Ensure PostgreSQL data & init directories exist + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: '0755' + loop: + - /opt/guacamole-db/data + - /opt/guacamole-db/initdb + +- name: Check Docker service status + ansible.builtin.service_facts: + +- name: Ensure Docker service is running + ansible.builtin.service: + name: docker + state: started + enabled: yes + register: docker_service_status + +- name: Wait for Docker daemon to be ready + ansible.builtin.wait_for: + path: /var/run/docker.sock + timeout: 30 + +- name: Test Docker connectivity + ansible.builtin.command: docker version + register: docker_test + retries: 5 + delay: 2 + until: docker_test.rc == 0 + +- name: Enable pgcrypto extension for initdb + ansible.builtin.copy: + dest: /opt/guacamole-db/initdb/00-enable-pgcrypto.sql + owner: root + group: root + mode: '0644' + content: | + -- Make digest(), hmac(), etc. available + CREATE EXTENSION IF NOT EXISTS pgcrypto; + +- name: Template per-user Guacamole user SQL + ansible.builtin.template: + src: user_bootstrap.sql.j2 + dest: "/opt/guacamole-db/initdb/02-{{ item.username }}_user.sql" + owner: root + group: root + mode: '0644' + loop: "{{ vdi_users_updated }}" + loop_control: + loop_var: item + vars: + username: "{{ item.username }}" + vdi_password: "{{ item.password }}" + +- name: Template per-user SQL for Guacamole connections + ansible.builtin.template: + src: connection_bootstrap.sql.j2 + dest: "/opt/guacamole-db/initdb/03-{{ item.username }}_connection.sql" + owner: root + group: root + mode: '0644' + loop: "{{ vdi_users_updated }}" + loop_control: + loop_var: item + vars: + port: "{{ item.port }}" + username: "{{ item.username }}" + vnc_password: "{{ item.vncserver_password }}" + +- name: Generate combined initdb.sql + ansible.builtin.shell: | + docker run --rm \ + -v /opt/guacamole-db/initdb:/opt/guacamole-db/initdb \ + guacamole/guacamole:latest \ + /opt/guacamole/bin/initdb.sh --postgresql \ + > /opt/guacamole-db/initdb/01-initdb.sql + args: + creates: /opt/guacamole-db/initdb/01-initdb.sql + +- name: Update guacadmin password hash in 01-initdb.sql + ansible.builtin.replace: + path: /opt/guacamole-db/initdb/01-initdb.sql + regexp: "decode\\('CA458A7D494E3BE824F5E1E175A1556C0F8EEF2C2D7DF3633BEC4A29C4411960', 'hex'\\)" + replace: "decode('{{ webapp_admin_hash }}', 'hex')" + +- name: Remove salt from guacadmin in 01-initdb.sql + ansible.builtin.replace: + path: /opt/guacamole-db/initdb/01-initdb.sql + regexp: "decode\\('FE24ADC5E11E2B25288D1704ABE67A79E342ECC26064CE69C5B3177795A82264', 'hex'\\)" + replace: "NULL" + +- name: Pull required Docker images + ansible.builtin.command: docker pull {{ item }} + loop: + - postgres:latest + - guacamole/guacd:latest + - guacamole/guacamole:latest + +- name: Create a dedicated Docker network + ansible.builtin.command: docker network create guac_net + register: network_create + failed_when: + - network_create.rc != 0 + - "'already exists' not in network_create.stderr" + +- name: Start Guacamole PostgreSQL container + ansible.builtin.command: > + docker run -d --name guac_db --network guac_net + -p 5432:5432 + -e POSTGRES_USER=guacamole_db + -e POSTGRES_PASSWORD={{ database_password }} + -e POSTGRES_DB=guacamole_db + -v /opt/guacamole-db/data:/var/lib/postgresql/data + -v /opt/guacamole-db/initdb:/docker-entrypoint-initdb.d + --restart always + postgres:latest + register: postgres_start + failed_when: + - postgres_start.rc != 0 + - "'already in use' not in postgres_start.stderr" + +- name: Wait for PostgreSQL on host port + ansible.builtin.wait_for: + host: 127.0.0.1 + port: 5432 + delay: 2 + timeout: 60 + +- name: Start guacd container + ansible.builtin.command: > + docker run -d --name guacd --network host + --restart always + guacamole/guacd:latest + register: guacd_start + failed_when: + - guacd_start.rc != 0 + - "'already in use' not in guacd_start.stderr" + +- name: Start Guacamole webapp container + ansible.builtin.command: > + docker run -d --name guac_app --network host + -e POSTGRES_HOSTNAME=127.0.0.1 + -e POSTGRES_PORT=5432 + -e POSTGRES_DATABASE=guacamole_db + -e POSTGRES_USER=guacamole_db + -e POSTGRES_PASSWORD={{ database_password }} + -e POSTGRES_AUTO_CREATE_ACCOUNTS=true + -e GUACD_HOSTNAME=localhost + -e GUACD_PORT=4822 + --restart always + guacamole/guacamole:latest + register: guac_app_start + failed_when: + - guac_app_start.rc != 0 + - "'already in use' not in guac_app_start.stderr" + +- name: Wait for Guacamole HTTP endpoint + ansible.builtin.wait_for: + host: 127.0.0.1 + port: "{{ vdi_webapp_port }}" + delay: 2 + timeout: 60 + +- name: Check Guacamole UI is up + ansible.builtin.uri: + url: "http://localhost:{{ vdi_webapp_port }}/guacamole/" + status_code: 200 + return_content: no + register: guac_health + until: guac_health.status == 200 + retries: 10 + delay: 3 + +- name: Obtain Guacamole API token to verify login + ansible.builtin.uri: + url: "http://localhost:{{ vdi_webapp_port }}/guacamole/api/tokens" + method: POST + body_format: form-urlencoded + body: + username: "guacadmin" + password: "{{ webapp_admin_password }}" + return_content: yes + register: guac_api + until: + - guac_api.status == 200 + - guac_api.json.authToken is defined + retries: 10 + delay: 3 + +- name: Remove bootstrap SQL files unless debug is enabled + ansible.builtin.file: + path: /opt/guacamole-db/initdb + state: absent + when: not debug | default(false) diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml new file mode 100644 index 0000000000..4257eb3b52 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml @@ -0,0 +1,26 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +- name: "Install Guacamole" + include_tasks: guacamole.yaml + when: vdi_tool | lower == 'guacamole' + +- name: "Install NoMachine" + include_tasks: nomachine.yaml + when: vdi_tool | lower == 'nomachine' + +- name: "Install Workspot" + include_tasks: workspot.yaml + when: vdi_tool | lower == 'workspot' diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/nomachine.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/nomachine.yaml new file mode 100644 index 0000000000..3039c31d62 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/nomachine.yaml @@ -0,0 +1,14 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/workspot.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/workspot.yaml new file mode 100644 index 0000000000..3039c31d62 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/workspot.yaml @@ -0,0 +1,14 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/connection_bootstrap.sql.j2 b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/connection_bootstrap.sql.j2 new file mode 100644 index 0000000000..e90562fccc --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/connection_bootstrap.sql.j2 @@ -0,0 +1,72 @@ +-- VNC Connection for {{ username }} +WITH + user_entity AS ( + SELECT entity_id + FROM guacamole_entity + WHERE name = '{{ username }}' AND type = 'USER' + ), + new_connection AS ( + INSERT INTO guacamole_connection (connection_name, protocol) + VALUES ('{{ username }}', 'vnc') + RETURNING connection_id + ), + insert_params AS ( + INSERT INTO guacamole_connection_parameter (connection_id, parameter_name, parameter_value) + SELECT connection_id, param_name, param_value + FROM new_connection, (VALUES + ('hostname', '{{ ansible_default_ipv4.address }}'), + ('port', '{{ port }}'), + ('username', '{{ username }}'), + ('password', '{{ vnc_password }}') + ) AS params(param_name, param_value) + ), + insert_permissions AS ( + INSERT INTO guacamole_connection_permission (entity_id, connection_id, permission) + SELECT + user_entity.entity_id, + new_connection.connection_id, + perms.permission::guacamole_object_permission_type + FROM user_entity, new_connection, + (VALUES ('READ'),('UPDATE'),('DELETE'),('ADMINISTER')) AS perms(permission) + ) +SELECT 1; + +-- SSH Connection for {{ username }} +WITH + user_entity AS ( + SELECT entity_id + FROM guacamole_entity + WHERE name = '{{ username }}' AND type = 'USER' + ), + new_connection_ssh AS ( + INSERT INTO guacamole_connection (connection_name, protocol) + VALUES ('{{ username }} SSH', 'ssh') + RETURNING connection_id + ), + insert_params_ssh AS ( + INSERT INTO guacamole_connection_parameter (connection_id, parameter_name, parameter_value) + SELECT connection_id, param_name, param_value + FROM new_connection_ssh, (VALUES + ('hostname', '{{ ansible_default_ipv4.address }}'), + ('port', '22'), + ('username', '{{ username }}'), + ( + 'private-key', + E'{{ lookup("file", "/home/" ~ username ~ "/.ssh/id_rsa") + | replace("\\", "\\\\") + | replace("'", "''") + | replace("\r", "") + | replace("\n", "\\n") }}' + ) + ) AS params(param_name, param_value) + ), + insert_permissions_ssh AS ( + INSERT INTO guacamole_connection_permission (entity_id, connection_id, permission) + SELECT + user_entity.entity_id, + new_connection_ssh.connection_id, + perms.permission::guacamole_object_permission_type + FROM user_entity, new_connection_ssh, + (VALUES ('READ'),('UPDATE'),('DELETE'),('ADMINISTER')) AS perms(permission) + ) +SELECT 1; diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/user_bootstrap.sql.j2 b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/user_bootstrap.sql.j2 new file mode 100644 index 0000000000..fb6fc25571 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/user_bootstrap.sql.j2 @@ -0,0 +1,22 @@ +-- Create Guacamole entity + user for {{ username }} +WITH new_entity AS ( + INSERT INTO guacamole_entity (name, type) + VALUES ('{{ username }}', 'USER') + RETURNING entity_id +) +INSERT INTO guacamole_user ( + entity_id, + password_salt, + password_hash, + password_date, + disabled, + expired +) +SELECT + entity_id, + NULL, + digest('{{ vdi_password }}', 'SHA-256'), + now(), + false, + false +FROM new_entity; diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml new file mode 100644 index 0000000000..bc81213782 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml @@ -0,0 +1,23 @@ + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +- name: "Install TigerVNC" + include_tasks: tigervnc.yaml + when: vnc_flavor | lower == 'tigervnc' + +- name: "Install TightVNC" + include_tasks: tightvnc.yaml + when: vnc_flavor | lower == 'tightvnc' diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/tigervnc.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/tigervnc.yaml new file mode 100644 index 0000000000..0401806493 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/tigervnc.yaml @@ -0,0 +1,69 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# Per-user configuration +- name: Configure per-user VNC settings + include_tasks: vnc_user_config.yaml + loop: "{{ vdi_users_updated }}" + loop_control: + loop_var: vdi_user + +# Global VNC users mapping (for systemd service) +- name: Template TigerVNC users file + ansible.builtin.template: + src: vncserver.users.j2 + dest: /etc/tigervnc/vncserver.users + mode: '0644' + vars: + guac_map: "{{ vdi_users_updated }}" + +# Systemd units +- name: Deploy per-user TigerVNC systemd unit (Ubuntu/Debian) + ansible.builtin.template: + src: vncserver@.service.ubuntu.j2 + dest: "/etc/systemd/system/vncserver@{{ item.username }}.service" + mode: '0644' + vars: + display_number: "{{ item.display_number }}" + when: ansible_distribution in ["Ubuntu", "Debian"] + loop: "{{ vdi_users_updated }}" + +- name: Deploy TigerVNC systemd unit (Rocky) + ansible.builtin.template: + src: vncserver@.service.rocky.j2 + dest: /etc/systemd/system/vncserver@.service + mode: '0644' + when: ansible_os_family == "RedHat" + +- name: Reload systemd daemon + ansible.builtin.systemd: + daemon_reload: yes + +# Enable & start each user's VNC server service +- name: Enable & start TigerVNC (Ubuntu/Debian) + systemd: + name: "vncserver@{{ item.username }}" + enabled: yes + state: started + when: ansible_distribution in ["Ubuntu", "Debian"] + loop: "{{ vdi_users_updated }}" + +- name: Enable & start TigerVNC (Rocky) + systemd: + name: "vncserver@:{{ item.display_number }}" + enabled: yes + state: started + when: ansible_os_family == "RedHat" + loop: "{{ vdi_users_updated }}" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/tightvnc.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/tightvnc.yaml new file mode 100644 index 0000000000..3039c31d62 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/tightvnc.yaml @@ -0,0 +1,14 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/vnc_user_config.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/vnc_user_config.yaml new file mode 100644 index 0000000000..206551b270 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/vnc_user_config.yaml @@ -0,0 +1,59 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +- name: Ensure VNC configuration directory exists + ansible.builtin.file: + path: "/home/{{ vdi_user.username }}/.vnc" + state: directory + owner: "{{ vdi_user.username }}" + group: "{{ vdi_user_group }}" + mode: '0700' + +- name: Write VNC xstartup for {{ vdi_user.username }} (Ubuntu/Debian) + ansible.builtin.copy: + dest: "/home/{{ vdi_user.username }}/.vnc/xstartup" + content: | + #!/bin/sh + unset SESSION_MANAGER + unset DBUS_SESSION_BUS_ADDRESS + exec startxfce4 + owner: "{{ vdi_user.username }}" + group: "{{ vdi_user_group }}" + mode: '0755' + when: ansible_distribution in ["Ubuntu", "Debian"] + +- name: Write VNC password file for {{ vdi_user.username }} + ansible.builtin.shell: | + echo "{{ vdi_user.vncserver_password }}" | vncpasswd -f > "/home/{{ vdi_user.username }}/.vnc/passwd" + args: + creates: "/home/{{ vdi_user.username }}/.vnc/passwd" + become_user: "{{ vdi_user.username }}" + +- name: Fix passwd file permissions + ansible.builtin.file: + path: "/home/{{ vdi_user.username }}/.vnc/passwd" + owner: "{{ vdi_user.username }}" + group: "{{ vdi_user_group }}" + mode: '0600' + +- name: Copy per-user VNC config + ansible.builtin.template: + src: config.j2 + dest: "/home/{{ vdi_user.username }}/.vnc/config" + owner: "{{ vdi_user.username }}" + group: "{{ vdi_user_group }}" + mode: '0644' + vars: + vdi_resolution: "{{ vdi_resolution }}" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/config.j2 b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/config.j2 new file mode 100644 index 0000000000..605ec72eca --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/config.j2 @@ -0,0 +1,2 @@ +session=xfce +geometry={{ vdi_resolution }} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver.users.j2 b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver.users.j2 new file mode 100644 index 0000000000..cbdab2dbc0 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver.users.j2 @@ -0,0 +1,3 @@ +{% for user in guac_map %} +:{{ user.port - 5900 }}={{ user.username }} +{% endfor %} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.rocky.j2 b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.rocky.j2 new file mode 100644 index 0000000000..a7cc3cc10a --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.rocky.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=TigerVNC server for user %i +After=syslog.target network.target + +[Service] +Type=forking +ExecStart=/usr/libexec/vncsession-start %i +ExecStop=-/bin/true +PIDFile=/run/vncsession-%i.pid + +Restart=always +RestartSec=15 + +[Install] +WantedBy=multi-user.target diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.ubuntu.j2 b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.ubuntu.j2 new file mode 100644 index 0000000000..1df046f446 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.ubuntu.j2 @@ -0,0 +1,16 @@ +[Unit] +Description=TigerVNC server for user %i +After=network.target + +[Service] +Type=forking +User=%i +Environment=HOME=/home/%i +ExecStart=/usr/bin/vncserver -localhost no :{{ display_number }} +ExecStop=/usr/bin/vncserver -kill :{{ display_number }} + +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl new file mode 100644 index 0000000000..eed3983fb0 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl @@ -0,0 +1,12 @@ +# Generated by Cluster Toolkit + +- hosts: localhost + become: true + vars_files: + - "/tmp/vdi/vars.yaml" + tasks: + +%{ for r in roles } + - import_role: + name: "${r}" +%{ endfor } diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl new file mode 100644 index 0000000000..9960ecf9db --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl @@ -0,0 +1,23 @@ +# Generated by Cluster Toolkit + +deployment_name: ${deployment_name} +project_id: ${project_id} +user_provision: ${user_provision} +vnc_flavor: ${vnc_flavor} +vdi_tool: ${vdi_tool} +vdi_user_group: ${vdi_user_group} +vdi_webapp_port: ${vdi_webapp_port} +vdi_resolution: ${vdi_resolution} + +vdi_users: +%{ for user in vdi_users } + - username: ${user.username} + port: ${user.port} +%{ if user.secret_name != null } + secret_name: "${user.secret_name}" +%{ endif } +%{ if user.secret_project != null } + secret_project: "${user.secret_project}" +%{ endif } + +%{ endfor } diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/validation.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/validation.tf new file mode 100644 index 0000000000..02c3dfc822 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/validation.tf @@ -0,0 +1,53 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "terraform_data" "input_validation" { + lifecycle { + precondition { + condition = contains(["tigervnc"], var.vnc_flavor) + error_message = "vnc_flavor must be tigervnc." + } + + precondition { + condition = contains(["guacamole"], var.vdi_tool) + error_message = "vdi_tool must be one of: guacamole." + } + + precondition { + condition = contains(["local_users"], var.user_provision) + error_message = "user_provision must be local_users." + } + + precondition { + condition = can(regex("^[1-9][0-9]*x[1-9][0-9]*$", var.vdi_resolution)) + error_message = "vdi_resolution must be in the form WIDTHxHEIGHT (e.g. 1920x1080)." + } + + precondition { + condition = var.user_provision == "local_users" || length(var.vdi_users) == 0 + error_message = "vdi_users may only be set when user_provision = local_users." + } + + precondition { + condition = ( + var.vnc_flavor != "tigervnc" && var.vnc_flavor != "tightvnc" + ) || alltrue([ + for user in var.vdi_users : ( + user.port >= var.vnc_port_min && user.port <= var.vnc_port_max + ) + ]) + error_message = "Each VDI user must have a port between 5901 and 5999 when VNC is used." + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/variables.tf new file mode 100644 index 0000000000..38cf7bd42a --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/variables.tf @@ -0,0 +1,91 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "Project in which the HPC deployment will be created." + type = string +} + +variable "deployment_name" { + description = "The name of the deployment." + type = string +} + +variable "region" { + description = "Region to place bucket containing startup script." + type = string +} + +variable "labels" { + description = "Key-value pairs of labels to be added to created resources." + type = map(string) +} + +variable "vnc_flavor" { + description = "The VNC server flavor to use (tigervnc currently supported)" + type = string + default = "tigervnc" +} + +variable "vdi_tool" { + type = string + description = "VDI tool to deploy (guacamole currently supported)." + default = "guacamole" +} + +variable "user_provision" { + type = string + description = "User type to create (local_users supported. os-login to do." + default = "local_users" +} + +variable "vdi_user_group" { + type = string + description = "Unix group to create/use for VDI users." + default = "vdiusers" +} + +variable "vdi_resolution" { + type = string + description = "Desktop resolution for VNC sessions (e.g. 1920x1080)." + default = "1920x1080" +} + +variable "vdi_webapp_port" { + type = string + description = "Port to serve the Webapp interface from if applicable" + default = "8080" +} + +variable "vdi_users" { + description = "List of VDI users to configure. Passwords are handled securely by the Ansible roles: if secret_name is provided, the password is fetched from Secret Manager; if neither password nor secret_name is provided, a random password is generated and stored in Secret Manager. If secret_project is provided, it specifies the GCP project where the secret is stored (defaults to the deployment project)." + type = list(object({ + username = string + port = number + secret_name = optional(string) + secret_project = optional(string) + })) + default = [] +} + +variable "vnc_port_min" { + type = number + default = 5901 + description = "Minimum valid VNC port." +} +variable "vnc_port_max" { + type = number + default = 5999 + description = "Maximum valid VNC port." +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/versions.tf new file mode 100644 index 0000000000..6024b738a7 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.0" + required_providers { + archive = { + source = "hashicorp/archive" + version = "~> 2.0" + } + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/README.md new file mode 100644 index 0000000000..ee9c057c39 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/README.md @@ -0,0 +1,87 @@ +## Description + +This module will insert a dependency on the completion of the startup script +for one or more specified compute VMs and report back if it fails. This can be useful when running +post-boot installation scripts that require the startup script to finish setting up a node. + +> **_WARNING:_**: this module is experimental and not fully supported. + +### Additional Dependencies + +* [**gcloud**](https://cloud.google.com/sdk/gcloud) must be present in the path + of the machine where `terraform apply` is run. + +### Example + +```yaml +- id: workstation + source: modules/compute/vm-instance + use: + - network1 + - my-startup-script + settings: + instance_count: 4 + +# Wait for all instances of the above VM to finish running startup scripts. +- id: wait + source: community/modules/scripts/wait-for-startup + settings: + instance_names: $(workstation.name) +``` + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [null](#requirement\_null) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [null](#provider\_null) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.validate_instance_names](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.wait_for_startup](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | `""` | no | +| [instance\_name](#input\_instance\_name) | Name of the instance we are waiting for (can be null if 'instance\_names' is not empty) | `string` | `null` | no | +| [instance\_names](#input\_instance\_names) | A list of instance names we are waiting for, in addition to the one mentioned in 'instance\_name' (if any) | `list(string)` | `[]` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [timeout](#input\_timeout) | Timeout in seconds | `number` | `1200` | no | +| [zone](#input\_zone) | The GCP zone where the instance is running | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/main.tf new file mode 100644 index 0000000000..3f6b416251 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/main.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + combined_instance_names = concat(var.instance_names, [var.instance_name]) +} + +resource "null_resource" "validate_instance_names" { + lifecycle { + precondition { + condition = var.instance_name != null || length(var.instance_names) > 0 + error_message = "At least one instance name must be provided" + } + } +} + +resource "null_resource" "wait_for_startup" { + count = length(local.combined_instance_names) + + provisioner "local-exec" { + command = "/bin/bash ${path.module}/scripts/wait-for-startup-status.sh" + environment = { + INSTANCE_NAME = self.triggers.instance_name + ZONE = var.zone + PROJECT_ID = var.project_id + TIMEOUT = var.timeout + GCLOUD_PATH = var.gcloud_path_override + } + } + + triggers = { + instance_name = local.combined_instance_names[count.index] + } +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf new file mode 100644 index 0000000000..11a2ddf118 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf @@ -0,0 +1,15 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh new file mode 100644 index 0000000000..4a231f7def --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -z "${INSTANCE_NAME}" ]]; then + echo "INSTANCE_NAME is unset... exiting" + exit 0 +fi +if [[ -z "${ZONE}" ]]; then + echo "ZONE is unset" + exit 1 +fi +if [[ -z "${PROJECT_ID}" ]]; then + echo "PROJECT_ID is unset" + exit 1 +fi +if [[ -z "${TIMEOUT}" ]]; then + echo "TIMEOUT is unset" + exit 1 +fi + +if [[ -n "${GCLOUD_PATH}" ]]; then + export PATH="$GCLOUD_PATH:$PATH" +fi + +echo "Waiting for startup: instance_name='${INSTANCE_NAME}', zone='${ZONE}', project_id='${PROJECT_ID}', timeout_seconds='${TIMEOUT}'" + +# Wrapper around grep that swallows the error status code 1 +c1grep() { grep "$@" || test $? = 1; } + +now=$(date +%s) + +# If VM was created more than 30 days ago, serial port logs may no longer exist. +# Exit without errors if the instance is older than 30 days. +logsExpiryDays=30 +createdTimestampIso=$(gcloud compute instances describe "${INSTANCE_NAME}" --project "${PROJECT_ID}" --zone "${ZONE}" --format "value(creationTimestamp)") +earliestAllowedCreatedTimestamp=$(date -d "${createdTimestampIso} +${logsExpiryDays} day" +%s) +if [[ "$earliestAllowedCreatedTimestamp" -lt "$now" ]]; then + echo "Instance was created more than 30 days ago - serial port 1 logs are likely expired... exiting" + exit 0 +fi + +deadline=$((now + TIMEOUT)) +error_file=$(mktemp) +fetch_cmd="gcloud compute instances get-serial-port-output ${INSTANCE_NAME} --port 1 --zone ${ZONE} --project ${PROJECT_ID}" +# Match string for all finish types of the old guest agent and successful +# finishes on the new guest agent +FINISH_LINE="startup-script exit status" +# Match string for failures on the new guest agent +FINISH_LINE_ERR="Script.*failed with error:" + +NON_FATAL_ERRORS=( + "Internal error" +) + +until [[ now -gt deadline ]]; do + ser_log=$( + set -o pipefail + ${fetch_cmd} 2>"${error_file}" | + c1grep "${FINISH_LINE}\|${FINISH_LINE_ERR}" + ) || { + err=$(cat "${error_file}") + echo "$err" + fatal_error="true" + for e in "${NON_FATAL_ERRORS[@]}"; do + if [[ $err = *"$e"* ]]; then + fatal_error="false" + break + fi + done + + if [[ $fatal_error = "true" ]]; then + exit 1 + fi + } + if [[ -n "${ser_log}" ]]; then break; fi + echo "Could not detect end of startup script. Sleeping." + sleep 5 + now=$(date +%s) +done + +# This line checks for an exit code - the assumption is that there is a number +# at the end of the line and it is an exit code +STATUS=$(sed -r 's/.*([0-9]+)\s*$/\1/' <<<"${ser_log}" | uniq) +# This specific text is monitored for in tests, do not change. +INSPECT_OUTPUT_TEXT="To inspect the startup script output, please run:" +if [[ "${STATUS}" == 0 ]]; then + echo "startup-script finished successfully" +elif [[ "${STATUS}" == 1 ]]; then + echo "startup-script finished with errors, ${INSPECT_OUTPUT_TEXT}" + echo "${fetch_cmd}" +elif [[ now -ge deadline ]]; then + echo "startup-script timed out after ${TIMEOUT} seconds" + echo "${INSPECT_OUTPUT_TEXT}" + echo "${fetch_cmd}" + exit 1 +else + echo "Invalid return status: '${STATUS}'" + echo "${INSPECT_OUTPUT_TEXT}" + echo "${fetch_cmd}" + exit 1 +fi + +exit "${STATUS}" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf new file mode 100644 index 0000000000..fe6410a920 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "instance_name" { + description = "Name of the instance we are waiting for (can be null if 'instance_names' is not empty)" + type = string + default = null +} + +variable "instance_names" { + description = "A list of instance names we are waiting for, in addition to the one mentioned in 'instance_name' (if any)" + type = list(string) + default = [] +} + +variable "zone" { + description = "The GCP zone where the instance is running" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "timeout" { + description = "Timeout in seconds" + type = number + default = 1200 + validation { + condition = var.timeout >= 0 + error_message = "The timeout should be non-negative" + } +} + +variable "gcloud_path_override" { + description = "Directory of the gcloud executable to be used during cleanup" + type = string + default = "" + nullable = false +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf new file mode 100644 index 0000000000..bb5a32e08d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:wait-for-startup/v1.57.0" + } + + required_version = ">= 0.14.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/README.md new file mode 100644 index 0000000000..fc25bc0a55 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/README.md @@ -0,0 +1,109 @@ +## Description + +This module contains a set of scripts to be used in customizing Windows VMs at +boot or during image building. Please note that the installation of NVIDIA GPU +drivers takes, at minimum, 30-60 minutes. It is therefore recommended to build +a custom image and reuse it as shown below, rather than install GPU drivers at +boot time. + +> NOTE: the output `windows_startup_ps1` must be passed explicitly as shown +> below when used with Packer modules. This is due to a limitation in the `use` +> keyword and inputs of type `list` in Packer modules; this does not impact +> Terraform modules + +### NVIDIA Drivers and CUDA Toolkit + +Many Google Cloud VM families include or can have NVIDIA GPUs attached to them. +This module supports GPU applications by enabling you to easily install +a compatible release of NVIDIA drivers and of the CUDA Toolkit. The script is +the [solution recommended by our documentation][docs] and is [directly sourced +from GitHub][script-src]. + +[docs]: https://cloud.google.com/compute/docs/gpus/install-drivers-gpu#windows +[script-src]: https://github.com/GoogleCloudPlatform/compute-gpu-installation/blob/24dac3004360e0696c49560f2da2cd60fcb80107/windows/install_gpu_driver.ps1 + +```yaml +- group: primary + modules: + - id: network1 + source: modules/network/vpc + settings: + enable_iap_rdp_ingress: true + enable_iap_winrm_ingress: true + + - id: windows_startup + source: community/modules/scripts/windows-startup-script + settings: + install_nvidia_driver: true + +- group: packer + modules: + - id: image + source: modules/packer/custom-image + kind: packer + use: + - network1 + - windows_startup + settings: + source_image_family: windows-2016 + machine_type: n1-standard-8 + accelerator_count: 1 + accelerator_type: nvidia-tesla-t4 + disk_size: 75 + disk_type: pd-ssd + omit_external_ip: false + state_timeout: 15m +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [http\_proxy](#input\_http\_proxy) | Set http and https proxy for use by Invoke-WebRequest commands | `string` | `""` | no | +| [http\_proxy\_set\_environment](#input\_http\_proxy\_set\_environment) | Set system default environment variables http\_proxy and https\_proxy for all commands | `bool` | `false` | no | +| [install\_nvidia\_driver](#input\_install\_nvidia\_driver) | Install NVIDIA GPU drivers and the CUDA Toolkit using script specified by var.install\_nvidia\_driver\_script | `bool` | `false` | no | +| [install\_nvidia\_driver\_args](#input\_install\_nvidia\_driver\_args) | Arguments to supply to NVIDIA driver install script | `string` | `"/s /n"` | no | +| [install\_nvidia\_driver\_script](#input\_install\_nvidia\_driver\_script) | Install script for NVIDIA drivers specified by http/https URL | `string` | `"https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda_12.1.1_531.14_windows.exe"` | no | +| [no\_proxy](#input\_no\_proxy) | Environment variables no\_proxy (only used if var.http\_proxy\_set\_environment is enabled) | `string` | `"169.254.169.254,metadata,metadata.google.internal,.googleapis.com"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | A string list of scripts selected by this module | + diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/main.tf new file mode 100644 index 0000000000..5e6bc8b94d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/main.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + setx_http_proxy_ps1 = !var.http_proxy_set_environment ? [] : [ + templatefile("${path.module}/templates/setx_http_proxy.ps1", { + "http_proxy" : var.http_proxy, + "no_proxy" : var.no_proxy, + }) + ] + + nvidia_ps1 = !var.install_nvidia_driver ? [] : [ + templatefile("${path.module}/templates/install_gpu_driver.ps1.tftpl", { + "url" : var.install_nvidia_driver_script + "args" : var.install_nvidia_driver_args + "http_proxy" : var.http_proxy, + }) + ] + + startup_ps1 = concat(local.setx_http_proxy_ps1, local.nvidia_ps1) +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf new file mode 100644 index 0000000000..006ea312ad --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "windows_startup_ps1" { + description = "A string list of scripts selected by this module" + value = local.startup_ps1 +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl new file mode 100644 index 0000000000..55c4a2a3cd --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl @@ -0,0 +1,38 @@ +#Requires -RunAsAdministrator + +# Windows 2016 needs forced upgrade to TLS 1.2 +[Net.ServicePointManager]::SecurityProtocol = 'Tls12' + +# important for catching exception in Invoke-WebRequest +Set-StrictMode -Version latest +$ErrorActionPreference = 'Stop' + +%{ if http_proxy != "" } +[System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy("${http_proxy}") +%{ endif } + +# Create the folder for the driver download +$file_dir = 'C:\NVIDIA-Driver\nvidia_installer_windows.exe' +if (!(Test-Path -Path 'C:\NVIDIA-Driver')) { + New-Item -Path 'C:\' -Name 'NVIDIA-Driver' -ItemType 'directory' | Out-Null +} + +# Download the file to a specified directory +Write-Output "Downloading ${url} to $file_dir" +# Disabling progress bar has surprising large (10-100x) impact on speed +$ProgressPreference = 'SilentlyContinue' +try { + Invoke-WebRequest -Uri "${url}" -OutFile "$file_dir" +} catch { + Write-Output "$_" + throw "Failed to download ${url}; exiting startup script" +} + +# Install the file with the specified path from earlier as well as the RunAs admin option +Write-Output "Executing $file_dir with arguments '${args}'" +try { + Start-Process -FilePath "$file_dir" -ArgumentList '${args}' -Wait +} catch { + Write-Output "$_" + throw "Could not install NVIDIA driver; exiting startup script" +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 new file mode 100644 index 0000000000..e6d84b0681 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 @@ -0,0 +1,5 @@ +#Requires -RunAsAdministrator + +setx http_proxy ${http_proxy} /m +setx https_proxy ${http_proxy} /m +setx no_proxy ${no_proxy} /m diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf new file mode 100644 index 0000000000..9e4fb9e67d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "install_nvidia_driver" { + description = "Install NVIDIA GPU drivers and the CUDA Toolkit using script specified by var.install_nvidia_driver_script" + type = bool + default = false +} + +variable "install_nvidia_driver_script" { + description = "Install script for NVIDIA drivers specified by http/https URL" + type = string + default = "https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda_12.1.1_531.14_windows.exe" +} + +variable "install_nvidia_driver_args" { + description = "Arguments to supply to NVIDIA driver install script" + type = string + default = "/s /n" +} + +variable "http_proxy" { + description = "Set http and https proxy for use by Invoke-WebRequest commands" + type = string + default = "" + nullable = false +} + +variable "http_proxy_set_environment" { + description = "Set system default environment variables http_proxy and https_proxy for all commands" + type = bool + default = false + nullable = false +} + +variable "no_proxy" { + description = "Environment variables no_proxy (only used if var.http_proxy_set_environment is enabled)" + type = string + default = "169.254.169.254,metadata,metadata.google.internal,.googleapis.com" + nullable = false +} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf new file mode 100644 index 0000000000..9d5f6bd68d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:windows-startup-script/v1.57.0" + } + + required_version = ">= 0.14.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/README.md b/vdi-test-scott/primary/modules/embedded/modules/README.md new file mode 100644 index 0000000000..37831fc708 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/README.md @@ -0,0 +1,550 @@ +# Modules + +This directory contains a set of core modules built for the Cluster Toolkit. Modules +describe the building blocks of an AI/ML and HPC deployment. The expected fields in a +module are listed in more detail [below](#module-fields). Blueprints can be +extended in functionality by incorporating [modules from GitHub +repositories][ghmods]. + +[ghmods]: #github-modules + +## All Modules + +Modules from various sources are all listed here for visibility. Badges are used +to indicate the source and status of many of these resources. + +Modules listed below with the ![core-badge] badge are located in this +folder and are tested and maintained by the Cluster Toolkit team. + +Modules labeled with the ![community-badge] badge are contributed by +the community (including the Cluster Toolkit team, partners, etc.). Community modules +are located in the [community folder](../community/modules/README.md). + +Modules labeled with the ![deprecated-badge] badge are now deprecated and may be +removed in the future. Customers are advised to transition to alternatives. + +Modules that are still in development and less stable are labeled with the +![experimental-badge] badge. + +[core-badge]: https://img.shields.io/badge/-core-blue?style=plastic +[community-badge]: https://img.shields.io/badge/-community-%23b8def4?style=plastic +[stable-badge]: https://img.shields.io/badge/-stable-lightgrey?style=plastic +[experimental-badge]: https://img.shields.io/badge/-experimental-%23febfa2?style=plastic +[deprecated-badge]: https://img.shields.io/badge/-deprecated-%23fea2a2?style=plastic + +### Compute + +* **[vm-instance]** ![core-badge] : Creates one or more VM instances. +* **[schedmd-slurm-gcp-v6-partition]** ![core-badge] : + Creates a partition to be used by a [slurm-controller][schedmd-slurm-gcp-v6-controller]. +* **[schedmd-slurm-gcp-v6-nodeset]** ![core-badge] : + Creates a nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module. +* **[schedmd-slurm-gcp-v6-nodeset-tpu]** ![core-badge] : + Creates a TPU nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module. +* **[schedmd-slurm-gcp-v6-nodeset-dynamic]** ![core-badge] ![experimental-badge]: + Creates a dynamic nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module and instance template. +* **[gke-node-pool]** ![core-badge] ![experimental-badge] : Creates a + Kubernetes node pool using GKE. +* **[resource-policy]** ![core-badge] ![experimental-badge] : Create a resource policy for compute engines that can be applied to gke-node-pool's nodes. +* **[gke-job-template]** ![core-badge] ![experimental-badge] : Creates a + Kubernetes job file to be used with a [gke-node-pool]. +* **[htcondor-execute-point]** ![community-badge] ![experimental-badge] : + Manages a group of execute points for use in an [HTCondor + pool][htcondor-setup]. +* **[mig]** ![community-badge] ![experimental-badge] : Creates a Managed Instance Group. +* **[notebook]** ![community-badge] ![experimental-badge] : Creates a Vertex AI + Notebook. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. + +[vm-instance]: compute/vm-instance/README.md +[gke-node-pool]: ../modules/compute/gke-node-pool/README.md +[resource-policy]: ../modules/compute/resource-policy/README.md +[gke-job-template]: ../modules/compute/gke-job-template/README.md +[schedmd-slurm-gcp-v6-partition]: ../community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md +[schedmd-slurm-gcp-v6-nodeset]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md +[schedmd-slurm-gcp-v6-nodeset-tpu]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/README.md +[schedmd-slurm-gcp-v6-nodeset-dynamic]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/README.md +[htcondor-execute-point]: ../community/modules/compute/htcondor-execute-point/README.md +[mig]: ../community/modules/compute/mig/README.md +[notebook]: ../community/modules/compute/notebook/README.md +[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md + +### Database + +* **[slurm-cloudsql-federation]** ![community-badge] ![experimental-badge] : + Creates a [Google SQL Instance](https://cloud.google.com/sql/) meant to be + integrated with a [slurm-controller][schedmd-slurm-gcp-v6-controller]. +* **[bigquery-dataset]** ![community-badge] ![experimental-badge] : Creates a BQ + dataset. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. +* **[bigquery-table]** ![community-badge] ![experimental-badge] : Creates a BQ + table. Primarily used for + [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. + +[slurm-cloudsql-federation]: ../community/modules/database/slurm-cloudsql-federation/README.md +[bigquery-dataset]: ../community/modules/database/bigquery-dataset/README.md +[bigquery-table]: ../community/modules/database/bigquery-table/README.md +[fsi-montecarlo-on-batch]: ../community/modules/files/fsi-montecarlo-on-batch/README.md + +### File System + +* **[filestore]** ![core-badge] : Creates a + [filestore](https://cloud.google.com/filestore) file system. +* **[parallelstore]** ![core-badge] ![experimental-badge]: Creates a + [parallelstore](https://cloud.google.com/parallelstore) file system. +* **[pre-existing-network-storage]** ![core-badge] : Specifies a + pre-existing file system that can be mounted on a VM. +* **[managed-lustre]** ![core-badge] ![experimental-badge]: Creates a + [managed-lustred](https://cloud.google.com/managed-lustre) file system. +* **[DDN-EXAScaler]** ![community-badge] ![deprecated-badge] : Creates + a [DDN EXAscaler lustre](https://www.ddn.com/partners/google-cloud-platform/) + file system. This module is deprecated and will be removed by July 1, 2025. Consider migrating to managed-lustre. +* **[cloud-storage-bucket]** ![community-badge] ![experimental-badge] : Creates + a Google Cloud Storage (GCS) bucket. +* **[gke-persistent-volume]** ![core-badge] ![experimental-badge] : Creates + persistent volumes and persistent volume claims for shared storage. +* **[nfs-server]** ![community-badge] ![experimental-badge] : Creates a VM and + configures an NFS server that can be mounted by other VM. + +[filestore]: file-system/filestore/README.md +[parallelstore]: file-system/parallelstore/README.md +[pre-existing-network-storage]: file-system/pre-existing-network-storage/README.md +[managed-lustre]: file-system/managed-lustre/README.md +[ddn-exascaler]: ../community/modules/file-system/DDN-EXAScaler/README.md +[nfs-server]: ../community/modules/file-system/nfs-server/README.md +[cloud-storage-bucket]: ../community/modules/file-system/cloud-storage-bucket/README.md +[gke-persistent-volume]: ../modules/file-system/gke-persistent-volume/README.md + +### Monitoring + +* **[dashboard]** ![core-badge] : Creates a + [monitoring dashboard](https://cloud.google.com/monitoring/dashboards) for + visually tracking a Cluster Toolkit deployment. + +[dashboard]: monitoring/dashboard/README.md + +### Network + +* **[vpc]** ![core-badge] : Creates a + [Virtual Private Cloud (VPC)](https://cloud.google.com/vpc) network with + regional subnetworks and firewall rules. +* **[multivpc]** ![core-badge] ![experimental-badge]: Creates a variable + number of VPC networks using the [vpc] module. +* **[pre-existing-vpc]** ![core-badge] : Used to connect newly + built components to a pre-existing VPC network. +* **[firewall-rules]** ![core-badge] ![experimental-badge] : Add custom firewall + rules to existing networks (commonly used with [pre-existing-vpc]). +* **[private-service-access]** ![community-badge] ![experimental-badge] : + Configures Private Services Access for a VPC network (commonly used with [filestore] and [slurm-cloudsql-federation]). + +[vpc]: network/vpc/README.md +[multivpc]: network/multivpc/README.md +[pre-existing-vpc]: network/pre-existing-vpc/README.md +[firewall-rules]: network/firewall-rules/README.md +[private-service-access]: ../community/modules/network/private-service-access/README.md + +### Packer + +* **[custom-image]** ![core-badge] : Creates a custom VM Image + based on the GCP HPC VM image. + +[custom-image]: packer/custom-image/README.md + +### Project + +* **[service-account]** ![community-badge] ![experimental-badge] : Creates [service + accounts](https://cloud.google.com/iam/docs/service-accounts) for a GCP + project. +* **[service-enablement]** ![community-badge] ![experimental-badge] : Allows enabling + various APIs for a Google Cloud Project. + +[service-account]: ../community/modules/project/service-account/README.md +[service-enablement]: ../community/modules/project/service-enablement/README.md + +### Pub/Sub + +* **[topic]** ![community-badge] ![experimental-badge] : Creates a +Pub/Sub topic. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. +* **[bigquery-sub]** ![community-badge] ![experimental-badge] : Creates a +Pub/Sub subscription. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. + +[topic]: ../community/modules/pubsub/topic/README.md +[bigquery-sub]: ../community/modules/pubsub/bigquery-sub/README.md + +### Remote Desktop + +* **[chrome-remote-desktop]** ![community-badge] ![experimental-badge] : Creates + a GPU accelerated Chrome Remote Desktop. + +[chrome-remote-desktop]: ../community/modules/remote-desktop/chrome-remote-desktop/README.md + +### Scheduler + +* **[batch-job-template]** ![core-badge] : Creates a Google Cloud Batch job + template that works with other Toolkit modules. +* **[batch-login-node]** ![core-badge] : Creates a VM that can be used for + submission of Google Cloud Batch jobs. +* **[gke-cluster]** ![core-badge] ![experimental-badge] : Creates a + Kubernetes cluster using GKE. +* **[pre-existing-gke-cluster]** ![core-badge] ![experimental-badge] : Retrieves an existing GKE cluster. Substitute for ([gke-cluster]) module. +* **[schedmd-slurm-gcp-v6-controller]** ![core-badge] : + Creates a Slurm controller node. +* **[schedmd-slurm-gcp-v6-login]** ![core-badge] : + Creates a Slurm login node. +* **[htcondor-setup]** ![community-badge] ![experimental-badge] : Creates the + base infrastructure for an HTCondor pool (service accounts and Cloud Storage bucket). +* **[htcondor-pool-secrets]** ![community-badge] ![experimental-badge] : Creates + and manages access to the secrets necessary for secure operation of an + HTCondor pool. +* **[htcondor-access-point]** ![community-badge] ![experimental-badge] : Creates + a regional instance group managing a highly available HTCondor access point + (login node). + +[batch-job-template]: ../modules/scheduler/batch-job-template/README.md +[batch-login-node]: ../modules/scheduler/batch-login-node/README.md +[gke-cluster]: ../modules/scheduler/gke-cluster/README.md +[pre-existing-gke-cluster]: ../modules/scheduler/pre-existing-gke-cluster/README.md +[htcondor-setup]: ../community/modules/scheduler/htcondor-setup/README.md +[htcondor-pool-secrets]: ../community/modules/scheduler/htcondor-pool-secrets/README.md +[htcondor-access-point]: ../community/modules/scheduler/htcondor-access-point/README.md +[schedmd-slurm-gcp-v6-controller]: ../community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md +[schedmd-slurm-gcp-v6-login]: ../community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md + +### Scripts + +* **[startup-script]** ![core-badge] : Creates a customizable startup script + that can be fed into compute VMs. +* **[windows-startup-script]** ![community-badge] ![experimental-badge]: Creates + Windows PowerShell (PS1) scripts that can be used to customize Windows VMs + and VM images. +* **[htcondor-install]** ![community-badge] ![experimental-badge] : Creates + a startup script to install HTCondor and exports a list of required APIs +* **[ramble-execute]** ![community-badge] ![experimental-badge] : Creates a + startup script to execute + [Ramble](https://github.com/GoogleCloudPlatform/ramble) commands on a target + VM +* **[ramble-setup]** ![community-badge] ![experimental-badge] : Creates a + startup script to install + [Ramble](https://github.com/GoogleCloudPlatform/ramble) on an instance or a + slurm login or controller. +* **[spack-setup]** ![community-badge] ![experimental-badge] : Creates a startup + script to install [Spack](https://github.com/spack/spack) on an instance or a + slurm login or controller. +* **[spack-execute]** ![community-badge] ![experimental-badge] : Defines a + software build using [Spack](https://github.com/spack/spack). +* **[wait-for-startup]** ![community-badge] ![experimental-badge] : Waits for + successful completion of a startup script on a compute VM. + +[startup-script]: scripts/startup-script/README.md +[windows-startup-script]: ../community/modules/scripts/windows-startup-script/README.md +[htcondor-install]: ../community/modules/scripts/htcondor-install/README.md +[kubernetes-operations]: ../community/modules/scripts/kubernetes-operations/README.md +[ramble-execute]: ../community/modules/scripts/ramble-execute/README.md +[ramble-setup]: ../community/modules/scripts/ramble-setup/README.md +[spack-setup]: ../community/modules/scripts/spack-setup/README.md +[spack-execute]: ../community/modules/scripts/spack-execute/README.md +[wait-for-startup]: ../community/modules/scripts/wait-for-startup/README.md + +## Module Fields + +### ID (Required) + +The `id` field is used to uniquely identify and reference a defined module. +ID's are used in [variables](../examples/README.md#variables) and become the +name of each module when writing the terraform `main.tf` file. They are also +used in the [use](#use-optional) and [outputs](#outputs-optional) lists +described below. + +For terraform modules, the ID will be rendered into the terraform module label +at the top level main.tf file. + +### Source (Required) + +The source is a path or URL that points to the source files for Packer or +Terraform modules. A source can either be a filesystem path or a URL to a git +repository: + +* Filesystem paths + * modules embedded in the `gcluster` executable + * modules in the local filesystem +* Remote modules using [Terraform URL syntax](https://developer.hashicorp.com/terraform/language/modules/sources) + * Hosted on [GitHub](https://developer.hashicorp.com/terraform/language/modules/sources#github) + * Google Cloud Storage [Buckets](https://developer.hashicorp.com/terraform/language/modules/sources#gcs-bucket) + * Generic [git repositories](https://developer.hashicorp.com/terraform/language/modules/sources#generic-git-repository) + + when modules are in a subdirectory of the git repository, a special + double-slash `//` notation can be required as described below + +An important distinction is that those URLs are natively supported by Terraform so +they are not copied to your deployment directory. Packer does not have native +support for git-hosted modules so the Toolkit will copy these modules into the +deployment folder on your behalf. + +#### Embedded Modules + +Embedded modules are added to the gcluster binary during compilation and cannot +be edited. To refer to embedded modules, set the source path to +`modules/<>` or `community/modules/<>`. + +The paths match the modules in the repository structure for [core modules](./) +and [community modules](../community/modules/). Because the modules are embedded +during compilation, your local copies may differ unless you recompile gcluster. + +For example, this example snippet uses the embedded pre-existing-vpc module: + +```yaml + - id: network1 + source: modules/network/pre-existing-vpc +``` + +#### Local Modules + +Local modules point to a module in the file system and can easily be edited. +They are very useful during module development. To use a local module, set +the source to a path starting with `/`, `./`, or `../`. For instance, the +following module definition refers the local pre-existing-vpc modules. + +```yaml + - id: network1 + source: modules/network/pre-existing-vpc +``` + +> **_NOTE:_** Relative paths (beginning with `.` or `..` must be relative to the +> working directory from which `gcluster` is executed. This example would have to be +> run from a local copy of the Cluster Toolkit repository. An alternative is to use +> absolute paths to modules. + +#### GitHub-hosted Modules and Packages + +To use a Terraform module available on GitHub, set the source to a path starting +with `github.com` (HTTPS) or `git@github.com` (SSH). For instance, the following +module definition sources the Toolkit vpc module: + +```yaml + - id: network1 + source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/network/vpc +``` + +This example uses the [double-slash notation][tfsubdir] (`//`) to indicate that +the Toolkit is a "package" of multiple modules whose root directory is the root +of the git repository. The remainder of the path indicates the sub-directory of +the vpc module. + +The example above uses the default `main` branch of the Toolkit. Specific +[revisions][tfrev] can be selected with any valid [git reference][gitref]. +(git branch, commit hash or tag). If the git reference is a tag or branch, we +recommend setting `&depth=1` to reduce the data transferred over the network. +This option cannot be set when the reference is a commit hash. The following +examples select the vpc module on the active `develop` branch and also an older +release of the filestore module: + +```yaml + - id: network1 + source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/network/vpc?ref=develop + ... + - id: homefs + source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/file-system/filestore?ref=v1.22.1&depth=1 +``` + +Because Terraform modules natively support this syntax, gcluster will not copy +GitHub-hosted modules into your deployment folder. Terraform will download them +into a hidden folder when you run `terraform init`. + +[tfrev]: https://www.terraform.io/language/modules/sources#selecting-a-revision +[gitref]: https://git-scm.com/book/en/v2/Git-Tools-Revision-Selection#_single_revisions +[tfsubdir]: https://www.terraform.io/language/modules/sources#modules-in-package-sub-directories + +##### GitHub-hosted Packer modules + +Packer does not natively support GitHub-hosted modules so `gcluster create` will +copy modules into your deployment folder. + +If the module uses `//` package notation, `gcluster create` will copy the entire +repository to the module path: `deployment_name/group_name/module_id`. However, +when `gcluster deploy` is invoked, it will run Packer from the subdirectory +`deployment_name/group_name/module_id/subdirectory/after/double_slash`. + +If the module does not use `//` package notation, `gcluster create` will copy +only the final directory in the path to `deployment_name/group_name/module_id`. + +In all cases, `gcluster create` will remove the `.git` directory from the packer +module to ensure that you can manage the entire deployment directory with its +own git versioning. + +##### GitHub over SSH + +Get module from GitHub over SSH: + +```yaml + - id: network1 + source: git@github.com:GoogleCloudPlatform/hpc-toolkit.git//modules/network/vpc +``` + +Specific versions can be selected as for HTTPS: + +```yaml + - id: network1 + source: git@github.com:GoogleCloudPlatform/hpc-toolkit.git//modules/network/vpc?ref=v1.22.1&depth=1 +``` + +##### Generic Git Modules + +To use a Terraform module available in a non-GitHub git repository such as +gitlab, set the source to a path starting `git::`. Two Standard git protocols +are supported, `git::https://` for HTTPS or `git::git@github.com` for SSH. + +Additional formatting and features after `git::` are identical to that of the +[GitHub Modules](#github-modules) described above. + +#### Google Cloud Storage Modules + +To use a Terraform module available in a Google Cloud Storage bucket, set the source +to a URL with the special `gcs::` prefix, followed by a [GCS bucket object URL](https://cloud.google.com/storage/docs/request-endpoints#typical). + +For example: `gcs::https://www.googleapis.com/storage/v1/BUCKET_NAME/PATH_TO_MODULE` + +### Kind (May be Required) + +`kind` refers to the way in which a module is deployed. Currently, `kind` can be +either `terraform` or `packer`. It must be specified for modules of type +`packer`. If omitted, it will default to `terraform`. + +### Settings (May Be Required) + +The settings field is a map that supplies any user-defined variables for each +module. Settings values can be simple strings, numbers or booleans, but can +also support complex data types like maps and lists of variable depth. These +settings will become the values for the variables defined in either the +`variables.tf` file for Terraform or `variable.pkr.hcl` file for Packer. + +For some modules, there are mandatory variables that must be set, +therefore `settings` is a required field in that case. In many situations, a +combination of sensible defaults, deployment variables and used modules can +populated all required settings and therefore the settings field can be omitted. + +### Use (Optional) + +The `use` field is a powerful way of linking a module to one or more other +modules. When a module "uses" another module, the outputs of the used +module are compared to the settings of the current module. If they have +matching names and the setting has no explicit value, then it will be set to +the used module's output. For example, see the following blueprint snippet: + +```yaml +modules: +- id: network1 + source: modules/network/vpc + +- id: workstation + source: modules/compute/vm-instance + use: [network1] + settings: + ... +``` + +In this snippet, the VM instance `workstation` uses the outputs of vpc +`network1`. + +In this case both `network_self_link` and `subnetwork_self_link` in the +[workstation settings](compute/vm-instance/README.md#Inputs) will be set +to `$(network1.network_self_link)` and `$(network1.subnetwork_self_link)` which +refer to the [network1 outputs](network/vpc/README#Outputs) +of the same names. + +The order of precedence that `gcluster` uses in determining when to infer a setting +value is in the following priority order: + +1. Explicitly set in the blueprint using the `settings` field +1. Output from a used module, taken in the order provided in the `use` list +1. Deployment variable (`vars`) of the same name +1. Default value for the setting + +> **_NOTE:_** See the +> [network storage documentation](./../docs/network_storage.md) for more +> information about mounting network storage file systems via the `use` field. + +### Outputs (Optional) + +The `outputs` field adds the output of individual Terraform modules to the +output of its deployment group. This enables the value to be available via +`terraform output`. This can useful for displaying the IP of a login node or +printing instructions on how to use a module, as we have in the +[monitoring dashboard module](monitoring/dashboard/README.md#Outputs). + +The outputs field is a lists that it can be in either of two formats: a string +equal to the name of the module output, or a map specifying the `name`, +`description`, and whether the value is `sensitive` and should be suppressed +from the standard output of Terraform commands. An example is shown below +that displays the internal and public IP addresses of a VM created by the +vm-instance module: + +```yaml + - id: vm + source: modules/compute/vm-instance + use: + - network1 + settings: + machine_type: e2-medium + outputs: + - internal_ip + - name: external_ip + description: "External IP of VM" + sensitive: true +``` + +The outputs shown after running Terraform apply will resemble: + +```text +Apply complete! Resources: 7 added, 0 changed, 0 destroyed. + +Outputs: + +external_ip_simplevm = +internal_ip_simplevm = [ + "10.128.0.19", +] +``` + +### Required Services (APIs) (optional) + +Each Toolkit module depends upon Google Cloud services ("APIs") being enabled +in the project used by the AI/ML and HPC environment. For example, the [creation of +VMs](compute/vm-instance/) requires the Compute Engine API +(compute.googleapis.com). The [startup-script](scripts/startup-script/) module +requires the Cloud Storage API (storage.googleapis.com) for storage of the +scripts themselves. Each module included in the Toolkit source code describes +its required APIs internally. The Toolkit will merge the requirements from all +modules and [automatically validate](../README.md#blueprint-validation) that all +APIs are enabled in the project specified by `$(vars.project_id)`. + +## Common Settings + +The following common naming conventions should be used to decrease the verbosity +needed to define a blueprint. This is intentional to allow multiple +modules to share inferred settings from deployment variables or from other +modules listed under the `use` field. + +For example, if all modules are to be created in a single region, that region +can be defined as a deployment variable named `region`, which is shared between +all modules without an explicit setting. Similarly, if many modules need to be +connected to the same VPC network, they all can add the vpc module ID to their +`use` list so that `network_self_link` would be inferred from that vpc module rather +than having to set it manually. + +* **project_id**: The GCP project ID in which to create the GCP resources. +* **deployment_name**: The name of the current deployment of a blueprint. This + can help to avoid naming conflicts of modules when multiple deployments are + created from the same blueprint. +* **region**: The GCP + [region](https://cloud.google.com/compute/docs/regions-zones) the module + will be created in. +* **zone**: The GCP [zone](https://cloud.google.com/compute/docs/regions-zones) + the module will be created in. +* **labels**: + [Labels](https://cloud.google.com/resource-manager/docs/creating-managing-labels) + added to the module. In order to include any module in advanced + monitoring, labels must be exposed. We strongly recommend that all modules + expose this variable. + +## Writing Custom Cluster Toolkit Modules + +Modules are flexible by design, however we define some [best practices](../docs/module-guidelines.md) when +creating a new module meant to be used with the Cluster Toolkit. diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/README.md b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/README.md new file mode 100644 index 0000000000..7c49141d1f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/README.md @@ -0,0 +1,130 @@ +## Description + +This module is used to create a Kubernetes job template file. + +The job template file can be submitted as is or used as a template for further +customization. Add the `instructions` output to a blueprint (as shown below) to +get instructions on how to use `kubectl` to submit the job. + +This module is designed to `use` one or more `gke-node-pool` modules. The job +will be configured to run on any of the specified node pools. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +### Example + +The following example creates a GKE job template file. + +```yaml + - id: job-template + source: modules/compute/gke-job-template + use: [compute_pool] + settings: + node_count: 3 + outputs: [instructions] +``` + +Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). + +### Storage Options + +This module natively supports: + +* Filestore as a shared file system between pods/nodes. +* Pod level ephemeral storage options: + * memory backed emptyDir + * local SSD backed emptyDir + * SSD persistent disk backed ephemeral volume + * balanced persistent disk backed ephemeral volume + +See the [storage-gke.yaml blueprint](../../../examples/storage-gke.yaml) and the +associated [documentation](../../../../examples/README.md#storage-gkeyaml--) for +examples of how to use Filestore and ephemeral storage with this module. + +### Requested Resources + +When one or more `gke-node-pool` modules are referenced with the `use` field. +The requested resources will be populated to achieve a 1 pod per node packing +while still leaving some headroom for required system pods. + +This functionality can be overridden by specifying the desired cpu requirement +using the `requested_cpu_per_pod` setting. + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.2 | +| [local](#requirement\_local) | >= 2.0.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [local](#provider\_local) | >= 2.0.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [local_file.job_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allocatable\_cpu\_per\_node](#input\_allocatable\_cpu\_per\_node) | The allocatable cpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field. | `list(number)` | 
[
  -1
]
| no | +| [allocatable\_gpu\_per\_node](#input\_allocatable\_gpu\_per\_node) | The allocatable gpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field. | `list(number)` | 
[
  -1
]
| no | +| [backoff\_limit](#input\_backoff\_limit) | Controls the number of retries before considering a Job as failed. Set to zero for shared fate. | `number` | `0` | no | +| [command](#input\_command) | The command and arguments for the container that run in the Pod. The command field corresponds to entrypoint in some container runtimes. | `list(string)` | 
[
  "hostname"
]
| no | +| [completion\_mode](#input\_completion\_mode) | Sets value of `completionMode` on the job. Default uses indexed jobs. See [documentation](https://kubernetes.io/blog/2021/04/19/introducing-indexed-jobs/) for more information | `string` | `"Indexed"` | no | +| [ephemeral\_volumes](#input\_ephemeral\_volumes) | Will create an emptyDir or ephemeral volume that is backed by the specified type: `memory`, `local-ssd`, `pd-balanced`, `pd-ssd`. `size_gb` is provided in GiB. | 
list(object({
    type       = string
    mount_path = string
    size_gb    = number
  }))
| `[]` | no | +| [has\_gpu](#input\_has\_gpu) | Indicates that the job should request nodes with GPUs. Typically supplied by a gke-node-pool module. | `list(bool)` | 
[
  false
]
| no | +| [image](#input\_image) | The container image the job should use. | `string` | `"debian"` | no | +| [k8s\_service\_account\_name](#input\_k8s\_service\_account\_name) | Kubernetes service account to run the job as. If null then no service account is specified. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to the GKE job template. Key-value pairs. | `map(string)` | n/a | yes | +| [machine\_family](#input\_machine\_family) | The machine family to use in the node selector (example: `n2`). If null then machine family will not be used as selector criteria. | `string` | `null` | no | +| [name](#input\_name) | The name of the job. | `string` | `"my-job"` | no | +| [node\_count](#input\_node\_count) | How many nodes the job should run in parallel. | `number` | `1` | no | +| [node\_pool\_names](#input\_node\_pool\_names) | A list of node pool names on which to run the job. Can be populated via `use` field. | `list(string)` | `[]` | no | +| [node\_selectors](#input\_node\_selectors) | A list of node selectors to use to place the job. | 
list(object({
    key   = string
    value = string
  }))
| `[]` | no | +| [persistent\_volume\_claims](#input\_persistent\_volume\_claims) | A list of objects that describes a k8s PVC that is to be used and mounted on the job. Generally supplied by the gke-persistent-volume module. | 
list(object({
    name          = string
    mount_path    = string
    mount_options = string
    is_gcs        = bool
  }))
| `[]` | no | +| [random\_name\_sufix](#input\_random\_name\_sufix) | Appends a random suffix to the job name to avoid clashes. | `bool` | `true` | no | +| [requested\_cpu\_per\_pod](#input\_requested\_cpu\_per\_pod) | The requested cpu per pod. If null, allocatable\_cpu\_per\_node will be used to claim whole nodes. If provided will override allocatable\_cpu\_per\_node. | `number` | `-1` | no | +| [requested\_gpu\_per\_pod](#input\_requested\_gpu\_per\_pod) | The requested gpu per pod. If null, allocatable\_gpu\_per\_node will be used to claim whole nodes. If provided will override allocatable\_gpu\_per\_node. | `number` | `-1` | no | +| [restart\_policy](#input\_restart\_policy) | Job restart policy. Only a RestartPolicy equal to `Never` or `OnFailure` is allowed. | `string` | `"Never"` | no | +| [security\_context](#input\_security\_context) | The security options the container should be run with. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ | 
list(object({
    key   = string
    value = string
  }))
| `[]` | no | +| [tolerations](#input\_tolerations) | Tolerations allow the scheduler to schedule pods with matching taints. Generally populated from gke-node-pool via `use` field. | 
list(object({
    key      = string
    operator = string
    value    = string
    effect   = string
  }))
| 
[
  {
    "effect": "NoSchedule",
    "key": "user-workload",
    "operator": "Equal",
    "value": "true"
  }
]
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Instructions for submitting the GKE job. | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/main.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/main.tf new file mode 100644 index 0000000000..835e3ca874 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/main.tf @@ -0,0 +1,168 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "gke-job-template", ghpc_role = "compute" }) +} + +locals { + # Start with the minimum cpu available of used node pools + min_allocatable_cpu = min(var.allocatable_cpu_per_node...) + full_node_cpu_request = ( + local.min_allocatable_cpu > 2 ? # if large enough + local.min_allocatable_cpu - 1 : # leave headroom for 1 cpu + local.min_allocatable_cpu / 2 + 0.1 # else take just over half + ) - (local.any_gcs ? 0.25 : 0) # save room for gcs side car + + cpu_request = ( + var.requested_cpu_per_pod >= 0 ? # if user supplied requested cpu + var.requested_cpu_per_pod : # then honor it + ( # else + local.min_allocatable_cpu >= 0 ? # if allocatable cpu was supplied + local.full_node_cpu_request : # then claim the full node + -1 # else do not set a limit + ) + ) + millicpu = floor(local.cpu_request * 1000) + cpu_request_string = local.millicpu >= 0 ? "${local.millicpu}m" : null + full_node_request = local.min_allocatable_cpu >= 0 && var.requested_cpu_per_pod < 0 + + memory_request_value = try(sum([for ed in var.ephemeral_volumes : + ed.size_gb + if ed.type == "memory" + ]), 0) + memory_request_string = local.memory_request_value > 0 ? "${local.memory_request_value}Gi" : null + + ephemeral_request_value = try(sum([for ed in var.ephemeral_volumes : + ed.size_gb + if ed.type == "local-ssd" + ]), 0) + ephemeral_request_string = local.ephemeral_request_value > 0 ? "${local.ephemeral_request_value}Gi" : null + + uses_local_ssd = anytrue([for ed in var.ephemeral_volumes : + ed.type == "local-ssd" + ]) + local_ssd_node_selector = local.uses_local_ssd ? [{ + key = "cloud.google.com/gke-ephemeral-storage-local-ssd" + value = "true" + }] : [] + + # Setup limit for GPUs per pod + min_allocatable_gpu = min(var.allocatable_gpu_per_node...) + min_allocatable_gpu_per_pod = local.min_allocatable_gpu > 0 ? local.min_allocatable_gpu : null + gpu_limit_per_pod = var.requested_gpu_per_pod > 0 ? var.requested_gpu_per_pod : local.min_allocatable_gpu_per_pod + gpu_limit_string = alltrue(var.has_gpu) ? tostring(local.gpu_limit_per_pod) : null + + empty_dir_volumes = [for ed in var.ephemeral_volumes : + { + name = replace(trim(ed.mount_path, "/"), "/", "-") + mount_path = ed.mount_path + size_limit = "${ed.size_gb}Gi" + in_memory = ed.type == "memory" + } + if contains(["memory", "local-ssd"], ed.type) + ] + + ephemeral_pd_volumes = [for pd in var.ephemeral_volumes : + { + name = replace(trim(pd.mount_path, "/"), "/", "-") + mount_path = pd.mount_path + storage_class_name = pd.type == "pd-ssd" ? "premium-rwo" : "standard-rwo" + storage = "${pd.size_gb}Gi" + } + if contains(["pd-balanced", "pd-ssd"], pd.type) + ] + + pvc_volumes = [for pvc in var.persistent_volume_claims : + { + name = replace(trim(pvc.mount_path, "/"), "/", "-") + mount_path = pvc.mount_path + claim_name = pvc.name + } + ] + + volume_mounts = [for v in concat(local.empty_dir_volumes, local.ephemeral_pd_volumes, local.pvc_volumes) : + { + name = v.name + mount_path = v.mount_path + } + ] + + suffix = var.random_name_sufix ? "-${random_id.resource_name_suffix.hex}" : "" + machine_family_node_selector = var.machine_family != null ? [{ + key = "cloud.google.com/machine-family" + value = var.machine_family + }] : [] + node_selectors = concat(local.machine_family_node_selector, local.local_ssd_node_selector, var.node_selectors) + + any_gcs = anytrue([for pvc in var.persistent_volume_claims : + pvc.is_gcs + ]) + + job_template_contents = templatefile( + "${path.module}/templates/gke-job-base.yaml.tftpl", + { + name = var.name + suffix = local.suffix + image = var.image + command = var.command + node_count = var.node_count + completion_mode = var.completion_mode + k8s_service_account_name = var.k8s_service_account_name + node_pool_names = var.node_pool_names + node_selectors = local.node_selectors + full_node_request = local.full_node_request + cpu_request = local.cpu_request_string + gpu_limit = local.gpu_limit_string + restart_policy = var.restart_policy + backoff_limit = var.backoff_limit + tolerations = distinct(var.tolerations) + security_context = var.security_context + labels = local.labels + + empty_dir_volumes = local.empty_dir_volumes + ephemeral_pd_volumes = local.ephemeral_pd_volumes + pvc_volumes = local.pvc_volumes + volume_mounts = local.volume_mounts + memory_request = local.memory_request_string + ephemeral_request = local.ephemeral_request_string + gcs_annotation = local.any_gcs + } + ) + + job_template_output_path = "${path.root}/${var.name}${local.suffix}.yaml" + +} + +resource "random_id" "resource_name_suffix" { + byte_length = 2 + keepers = { + timestamp = timestamp() + } +} + +resource "local_file" "job_template" { + content = local.job_template_contents + filename = local.job_template_output_path + + lifecycle { + precondition { + condition = local.any_gcs ? var.k8s_service_account_name != null : true + error_message = "When using GCS, a kubernetes service account with workload identity is required. gke-cluster module will perform this setup when var.configure_workload_identity_sa is set to true." + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/outputs.tf new file mode 100644 index 0000000000..adf78e936d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/outputs.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "instructions" { + description = "Instructions for submitting the GKE job." + value = <<-EOT + A GKE job file has been created locally at: + ${abspath(local.job_template_output_path)} + + Use the following commands to: + Submit your job: + kubectl create -f ${abspath(local.job_template_output_path)} + EOT +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl new file mode 100644 index 0000000000..431a519b9c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl @@ -0,0 +1,120 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ${name}${suffix} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + parallelism: ${node_count} + completions: ${node_count} + completionMode: ${completion_mode} + template: + %{~ if gcs_annotation ~} + metadata: + annotations: + gke-gcsfuse/volumes: "true" + %{~ endif ~} + spec: + %{~ if length(security_context) > 0 ~} + securityContext: + %{~ for context in security_context ~} + ${context.key}: ${context.value} + %{~ endfor ~} + %{~ endif ~} + %{~ if k8s_service_account_name != null ~} + serviceAccountName: ${k8s_service_account_name} + %{~ endif ~} + %{~ if length(node_pool_names) > 0 ~} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-nodepool + operator: In + values: + %{~ for node_pool in node_pool_names ~} + - ${node_pool} + %{~ endfor ~} + %{~ endif ~} + %{~ if length(node_selectors) > 0 ~} + nodeSelector: + %{~ for selector in node_selectors ~} + ${selector.key}: "${selector.value}" + %{~ endfor ~} + %{~ endif ~} + tolerations: + %{~ for toleration in tolerations ~} + - key: ${toleration.key} + operator: ${toleration.operator} + value: "${toleration.value}" + effect: ${toleration.effect} + %{~ endfor ~} + containers: + - name: ${name}-container + image: ${image} + command: + %{for s in command}- ${indent(8, yamlencode(s))}%{~ endfor } + %{~ if gpu_limit != null || cpu_request != null ~} + resources: + %{~ if gpu_limit != null ~} + limits: + # GPUs should only be specified as limits + # https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/ + nvidia.com/gpu: ${gpu_limit} + %{~ endif ~} + %{~ if cpu_request != null || memory_request != null || ephemeral_request != null ~} + requests: + %{~ if full_node_request ~} + # cpu request attempts full node per pod + %{~ endif ~} + %{~ if cpu_request != null ~} + cpu: ${cpu_request} + %{~ endif ~} + %{~ if memory_request != null ~} + memory: ${memory_request} + %{~ endif ~} + %{~ if ephemeral_request != null ~} + ephemeral-storage: ${ephemeral_request} + %{~ endif ~} + %{~ endif ~} + %{~ endif ~} + %{~ if length(volume_mounts) > 0 ~} + volumeMounts: + %{~ for v in volume_mounts ~} + - name: ${v.name} + mountPath: ${v.mount_path} + %{~ endfor ~} + %{~ endif ~} + %{~ if length(volume_mounts) > 0 ~} + volumes: + %{~ for ed in empty_dir_volumes ~} + - name: ${ed.name} + emptyDir: + sizeLimit: ${ed.size_limit} + %{~ if ed.in_memory ~} + medium: "Memory" + %{~ endif ~} + %{~ endfor ~} + %{~ for pd in ephemeral_pd_volumes ~} + - name: ${pd.name} + ephemeral: + volumeClaimTemplate: + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: ${pd.storage_class_name} + resources: + requests: + storage: ${pd.storage} + %{~ endfor ~} + %{~ for pvc in pvc_volumes ~} + - name: ${pvc.name} + persistentVolumeClaim: + claimName: ${pvc.claim_name} + %{~ endfor ~} + %{~ endif ~} + restartPolicy: ${restart_policy} + backoffLimit: ${backoff_limit} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/variables.tf new file mode 100644 index 0000000000..f849831bf4 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/variables.tf @@ -0,0 +1,187 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "name" { + description = "The name of the job." + type = string + default = "my-job" +} + +variable "node_count" { + description = "How many nodes the job should run in parallel." + type = number + default = 1 +} + +variable "completion_mode" { + description = "Sets value of `completionMode` on the job. Default uses indexed jobs. See [documentation](https://kubernetes.io/blog/2021/04/19/introducing-indexed-jobs/) for more information" + type = string + default = "Indexed" +} + +variable "command" { + description = "The command and arguments for the container that run in the Pod. The command field corresponds to entrypoint in some container runtimes." + type = list(string) + default = ["hostname"] +} + +variable "image" { + description = "The container image the job should use." + type = string + default = "debian" +} + +variable "k8s_service_account_name" { + description = "Kubernetes service account to run the job as. If null then no service account is specified." + type = string + default = null +} + +variable "node_pool_names" { + description = "A list of node pool names on which to run the job. Can be populated via `use` field." + type = list(string) + default = [] +} + +variable "allocatable_cpu_per_node" { + description = "The allocatable cpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field." + type = list(number) + default = [-1] +} + +variable "has_gpu" { + description = "Indicates that the job should request nodes with GPUs. Typically supplied by a gke-node-pool module." + type = list(bool) + default = [false] +} + +variable "requested_cpu_per_pod" { + description = "The requested cpu per pod. If null, allocatable_cpu_per_node will be used to claim whole nodes. If provided will override allocatable_cpu_per_node." + type = number + default = -1 +} + +variable "allocatable_gpu_per_node" { + description = "The allocatable gpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field." + type = list(number) + default = [-1] +} + +variable "requested_gpu_per_pod" { + description = "The requested gpu per pod. If null, allocatable_gpu_per_node will be used to claim whole nodes. If provided will override allocatable_gpu_per_node." + type = number + default = -1 +} + +variable "tolerations" { + description = "Tolerations allow the scheduler to schedule pods with matching taints. Generally populated from gke-node-pool via `use` field." + type = list(object({ + key = string + operator = string + value = string + effect = string + })) + default = [ + { + key = "user-workload" + operator = "Equal" + value = "true" + effect = "NoSchedule" + } + ] +} + +variable "security_context" { + description = "The security options the container should be run with. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/" + type = list(object({ + key = string + value = string + })) + default = [] +} + +variable "machine_family" { + description = "The machine family to use in the node selector (example: `n2`). If null then machine family will not be used as selector criteria." + type = string + default = null +} + +variable "node_selectors" { + description = "A list of node selectors to use to place the job." + type = list(object({ + key = string + value = string + })) + default = [] +} + +variable "restart_policy" { + description = "Job restart policy. Only a RestartPolicy equal to `Never` or `OnFailure` is allowed." + type = string + default = "Never" +} + +variable "backoff_limit" { + description = "Controls the number of retries before considering a Job as failed. Set to zero for shared fate." + type = number + default = 0 +} + +variable "random_name_sufix" { + description = "Appends a random suffix to the job name to avoid clashes." + type = bool + default = true +} + +variable "persistent_volume_claims" { + description = "A list of objects that describes a k8s PVC that is to be used and mounted on the job. Generally supplied by the gke-persistent-volume module." + type = list(object({ + name = string + mount_path = string + mount_options = string + is_gcs = bool + })) + default = [] +} + +variable "ephemeral_volumes" { + description = "Will create an emptyDir or ephemeral volume that is backed by the specified type: `memory`, `local-ssd`, `pd-balanced`, `pd-ssd`. `size_gb` is provided in GiB." + type = list(object({ + type = string + mount_path = string + size_gb = number + })) + default = [] + validation { + condition = alltrue([ + for v in var.ephemeral_volumes : + contains(["pd-balanced", "pd-ssd", "memory", "local-ssd"], v.type) + ]) + error_message = "Type must be one of 'pd-balanced', 'pd-ssd', 'memory', 'local-ssd'." + } + validation { + condition = alltrue([ + for v in var.ephemeral_volumes : + substr(v.mount_path, 0, 1) == "/" + ]) + error_message = "Mount path must start with the '/' character." + } +} + +variable "labels" { + description = "Labels to add to the GKE job template. Key-value pairs." + type = map(string) +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/versions.tf new file mode 100644 index 0000000000..0f902ac8c5 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/versions.tf @@ -0,0 +1,28 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.2" + + required_providers { + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/README.md b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/README.md new file mode 100644 index 0000000000..c8b0591544 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/README.md @@ -0,0 +1,375 @@ +## Description + +This module creates a Google Kubernetes Engine +([GKE](https://cloud.google.com/kubernetes-engine)) node pool. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +### Example + +The following example creates a GKE node group. + +```yaml + - id: compute_pool + source: modules/compute/gke-node-pool + use: [gke_cluster] +``` + +Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). + +### Taints and Tolerations + +By default node pools created with this module will be tainted with +`user-workload=true:NoSchedule` to prevent system pods from being scheduled. +User jobs targeting the node pool should include this toleration. This behavior +can be overridden using the `taints` setting. See +[docs](https://cloud.google.com/kubernetes-engine/docs/how-to/node-taints) for +more info. + +### Local SSD Storage +GKE offers two options for managing locally attached SSDs. + +The first, and recommended, option is for GKE to manage the ephemeral storage +space on the node, which will then be automatically attached to pods which +request an `emptyDir` volume. This can be accomplished using the +[`local_ssd_count_ephemeral_storage`] variable. + +The second, more complex, option is for GCP to attach these nodes as raw block +storage. In this case, the cluster administrator is responsible for software +RAID settings, partitioning, formatting and mounting these disks on the host +OS. Still, this may be desired behavior in use cases which aren't supported +by an `emptyDir` volume (for example, a `ReadOnlyMany` or `ReadWriteMany` PV). +This can be accomplished using the [`local_ssd_count_nvme_block`] variable. + +The [`local_ssd_count_ephemeral_storage`] and [`local_ssd_count_nvme_block`] +variables are mutually exclusive and cannot be mixed together. + +Also, the number of SSDs which can be attached to a node depends on the +[machine type](https://cloud.google.com/compute/docs/disks#local_ssd_machine_type_restrictions). + +See [docs](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/local-ssd) +for more info. + +[`local_ssd_count_ephemeral_storage`]: #input\_local\_ssd\_count\_ephemeral\_storage +[`local_ssd_count_nvme_block`]: #input\_local\_ssd\_count\_nvme\_block + +### Considerations with GPUs + +When a GPU is attached to a node an additional taint is automatically added: +`nvidia.com/gpu=present:NoSchedule`. For jobs to get placed on these nodes, the +equivalent toleration is required. The `gke-job-template` module will +automatically apply this toleration when using a node pool with GPUs. + +Nvidia GPU drivers must be installed. The recommended approach for GKE to install +GPU dirvers is by applying a DaemonSet to the cluster. See +[these instructions](https://cloud.google.com/kubernetes-engine/docs/how-to/gpus#cos). + +However, in some cases it may be desired to compile a different driver (such as +a desire to install a newer version, compatibility with the +[Nvidia GPU-operator](https://github.com/NVIDIA/gpu-operator) or other +use-cases). In this case, ensure that you turn off the +[enable_secure_boot](#input\_enable\_secure\_boot) option to allow unsigned +kernel modules to be loaded. + +#### Maximize GPU network bandwidth with GPUDirect and multi-networking +For A3 Series machines to achieve optimal performance , GKE provide two networking stacks for remote direct memory access (RDMA): + +- A3 High machine types (a3-highgpu-8g): utilize GPUDirect-TCPX to reduce the overhead required to transfer packet payloads to and from GPUs, which significantly improves throughput at scale compared to GPUs that don't use GPUDirect. +- A3 Mega machine types (a3-megagpu-8g): utilize GPUDirect-TCPXO to improve GPU to GPU communication, and further improves GPU to VM communication. + +To achieve this, when creating nodepools with A3 Series machine type, pass in a multivpc module to the gke-node-pool module, and the gke-node-pool module would detect the eligible machine type and enable GPUDirect for it. More specifically, the below components will be installed in the nodepool for enabling GPUDirect. + +- Install NCCL plugin for GPUDirect [TCPX](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/gpudirect-tcpx) or [TCPXO](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/gpudirect-tcpxo) +- Install [NRI](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/nri_device_injector) device injector plugin +- Provide support for injecting GPUDirect required components(annotations, volumes, rxdm sidecar etc.) into the user workload in the form of Kubernetes Job. + - Provide sample workload to showcase how it will be updated with the required components injected, and how it can be deployed. + - Allow user to use the provided script to update their own workload and deploy. + +The GPUDirect supports included in the Cluster Toolkit aim to automate the [GPUDirect User Guid](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#install-gpudirect-tcpx-nccl) and provide better usability. + +> **_NOTE:_** You must [enable multi networking](https://cloud.google.com/kubernetes-engine/docs/how-to/setup-multinetwork-support-for-pods#create-a-gke-cluster) feature when creating the GKE cluster. When gke-cluster depends on multivpc (with the use keyword), multi networking will be automatically enabled on the cluster creation. +> When gke-cluster or pre-existing-gke-cluster depends on multivpc (with the use keyword), the [network objects](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment) required for multi networking will be created on the cluster. + +### GPUs Examples + +There are several ways to add GPUs to a GKE node pool. See +[docs](https://cloud.google.com/compute/docs/gpus) for more info on GPUs. + +The following is a node pool that uses `a2`, `a3` or `g2` machine types which has a +fixed number of attached GPUs, let's call these machine types as "pre-defined gpu machine families": + +```yaml + - id: simple-a2-pool + source: modules/compute/gke-node-pool + use: [gke_cluster] + settings: + machine_type: a2-highgpu-1g +``` + +> **Note**: It is not necessary to define the [`guest_accelerator`] setting when +> using pre-defined gpu machine families as information about GPUs, such as type, count and +> `gpu_driver_installation_config`, is automatically inferred from the machine type. +> Optional fields such as `gpu_partition_size` need to be specified only if they have +> non-default values. + +The following scenarios require the [`guest_accelerator`] block is specified: + +- To partition an A100 GPU into multiple GPUs on an A2 family machine. +- To specify a time sharing configuration on a GPUs. +- To attach a GPU to an N1 family machine. + +The following is an example of +[partitioning](https://cloud.google.com/kubernetes-engine/docs/how-to/gpus-multi) +an A100 GPU: + +> **Note**: In the following example, `type`, `count` and `gpu_driver_installation_config` are picked up automatically. + +```yaml + - id: multi-instance-gpu-pool + source: modules/compute/gke-node-pool + use: [gke_cluster] + settings: + machine_type: a2-highgpu-1g + guest_accelerator: + - gpu_partition_size: 1g.5gb +``` + +[`guest_accelerator`]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_guest_accelerator + +The following is an example of +[GPU time sharing](https://cloud.google.com/kubernetes-engine/docs/concepts/timesharing-gpus) +(with partitioned GPUs): + +```yaml + - id: time-sharing-gpu-pool + source: modules/compute/gke-node-pool + use: [gke_cluster] + settings: + machine_type: a2-highgpu-1g + guest_accelerator: + - gpu_partition_size: 1g.5gb + gpu_sharing_config: + gpu_sharing_strategy: TIME_SHARING + max_shared_clients_per_gpu: 3 +``` + +Following is an example of using a GPU attached to an `n1` machine: + +```yaml + - id: t4-pool + source: modules/compute/gke-node-pool + use: [gke_cluster] + settings: + machine_type: n1-standard-16 + guest_accelerator: + - type: nvidia-tesla-t4 + count: 2 +``` + +The following is an example of using a GPU (with sharing config) attached to an `n1` machine: + +```yaml + - id: n1-t4-pool + source: community/modules/compute/gke-node-pool + use: [gke_cluster] + settings: + name: n1-t4-pool + machine_type: n1-standard-1 + guest_accelerator: + - type: nvidia-tesla-t4 + count: 2 + gpu_driver_installation_config: + gpu_driver_version: "LATEST" + gpu_sharing_config: + max_shared_clients_per_gpu: 2 + gpu_sharing_strategy: "TIME_SHARING" +``` + +Finally, the following is adding multivpc to a node pool: + +```yaml + - id: network + source: modules/network/vpc + settings: + subnetwork_name: gke-subnet + secondary_ranges: + gke-subnet: + - range_name: pods + ip_cidr_range: 10.4.0.0/14 + - range_name: services + ip_cidr_range: 10.0.32.0/20 + + - id: multinetwork + source: modules/network/multivpc + settings: + network_name_prefix: multivpc-net + network_count: 8 + global_ip_address_range: 172.16.0.0/12 + subnetwork_cidr_suffix: 16 + + - id: gke-cluster + source: modules/scheduler/gke-cluster + use: [network, multinetwork] + settings: + cluster_name: $(vars.deployment_name) + + - id: a3-megagpu_pool + source: modules/compute/gke-node-pool + use: [gke-cluster, multinetwork] + settings: + machine_type: a3-megagpu-8g + ... +``` + +## Using GCE Reservations +You can reserve Google Compute Engine instances in a specific zone to ensure resources are available for their workloads when needed. For more details on how to manage reservations, see [Reserving Compute Engine zonal resources](https://cloud.google.com/compute/docs/instances/reserving-zonal-resources). + +After creating a reservation, you can consume the reserved GCE VM instances in GKE. GKE clusters deployed using Cluster Toolkit support the same consumption modes as Compute Engine: NO_RESERVATION(default), ANY_RESERVATION, SPECIFIC_RESERVATION. + +This can be accomplished using [`reservation_affinity`](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/compute/gke-node-pool/README.md#input_reservation_affinity). + +```yaml +# Target any reservation +reservation_affinity: + consume_reservation_type: ANY_RESERVATION + +# Target a specific reservation +reservation_affinity: + consume_reservation_type: SPECIFIC_RESERVATION + specific_reservations: + - name: specific-reservation-1 +``` + +The following requirements need to be satisfied for the node pool nodes to be able to use a specific reservation: +1. A reservation with the name must exist in the specified project(`var.project_id`) and one of the specified zones(`var.zones`). +2. Its consumption type must be `specific`. +3. Its GCE VM Properties must match with those of the Node Pool; Machine type, Accelerators (GPU Type and count), Local SSD disk type and count. + +If you want to utilise a shared reservation, the owner project of the shared reservation needs to be explicitly specified like the following. Note that a shared reservation can be used by the project that hosts the reservation (owner project) and by the projects the reservation is shared with (consumer projects). See how to [create and use a shared reservation](https://cloud.google.com/compute/docs/instances/reservations-shared). + +```yaml +reservation_affinity: + consume_reservation_type: SPECIFIC_RESERVATION + specific_reservations: + - name: specific-reservation-shared + project: shared_reservation_owner_project_id +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | +| [google](#requirement\_google) | >= 6.41 | +| [null](#requirement\_null) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.41 | +| [null](#provider\_null) | ~> 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gpu](#module\_gpu) | ../../internal/gpu-definition | n/a | +| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_container_node_pool.node_pool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool) | resource | +| [null_resource.enable_tcpx_in_workload](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.enable_tcpxo_in_workload](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.install_dependencies](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_machine_types.machine_info](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_machine_types) | data source | +| [google_compute_reservation.specific_reservations](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_reservation) | data source | +| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks adds additional node networks to the node pool | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [auto\_repair](#input\_auto\_repair) | Whether the nodes will be automatically repaired. | `bool` | `true` | no | +| [auto\_upgrade](#input\_auto\_upgrade) | Whether the nodes will be automatically upgraded. | `bool` | `false` | no | +| [autoscaling\_total\_max\_nodes](#input\_autoscaling\_total\_max\_nodes) | Total maximum number of nodes in the NodePool. | `number` | `1000` | no | +| [autoscaling\_total\_min\_nodes](#input\_autoscaling\_total\_min\_nodes) | Total minimum number of nodes in the NodePool. | `number` | `0` | no | +| [cluster\_id](#input\_cluster\_id) | projects/{{project}}/locations/{{location}}/clusters/{{cluster}} | `string` | n/a | yes | +| [compact\_placement](#input\_compact\_placement) | DEPRECATED: Use `placement_policy` | `bool` | `null` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for each node. | `number` | `100` | no | +| [disk\_type](#input\_disk\_type) | Disk type for each node. | `string` | `null` | no | +| [enable\_flex\_start](#input\_enable\_flex\_start) | If true, start the node pool with Flex Start provisioning model.
To learn more about flex-start mode, please refer to
https://cloud.google.com/kubernetes-engine/docs/how-to/dws-flex-start-training and
https://cloud.google.com/kubernetes-engine/docs/how-to/provisioningrequest | `bool` | `false` | no | +| [enable\_gcfs](#input\_enable\_gcfs) | Enable the Google Container Filesystem (GCFS). See [restrictions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#gcfs_config). | `bool` | `false` | no | +| [enable\_private\_nodes](#input\_enable\_private\_nodes) | Whether nodes have internal IP addresses only. | `bool` | `false` | no | +| [enable\_queued\_provisioning](#input\_enable\_queued\_provisioning) | If true, enables Dynamic Workload Scheduler and adds the cloud.google.com/gke-queued taint to the node pool. | `bool` | `false` | no | +| [enable\_secure\_boot](#input\_enable\_secure\_boot) | Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info. | `bool` | `true` | no | +| [gke\_version](#input\_gke\_version) | GKE version | `string` | n/a | yes | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = optional(string)
    count = optional(number, 0)
    gpu_driver_installation_config = optional(object({
      gpu_driver_version = string
    }), { gpu_driver_version = "DEFAULT" })
    gpu_partition_size = optional(string)
    gpu_sharing_config = optional(object({
      gpu_sharing_strategy       = string
      max_shared_clients_per_gpu = number
    }))
  }))
| `[]` | no | +| [host\_maintenance\_interval](#input\_host\_maintenance\_interval) | Specifies the frequency of planned maintenance events. | `string` | `""` | no | +| [image\_type](#input\_image\_type) | The default image type used by NAP once a new node pool is being created. Use either COS\_CONTAINERD or UBUNTU\_CONTAINERD. | `string` | `"COS_CONTAINERD"` | no | +| [initial\_node\_count](#input\_initial\_node\_count) | The initial number of nodes for the pool. In regional clusters, this is the number of nodes per zone. Changing this setting after node pool creation will not make any effect. It cannot be set with static\_node\_count and must be set to a value between autoscaling\_total\_min\_nodes and autoscaling\_total\_max\_nodes. | `number` | `null` | no | +| [internal\_ghpc\_module\_id](#input\_internal\_ghpc\_module\_id) | DO NOT SET THIS MANUALLY. Automatically populates with module id (unique blueprint-wide). | `string` | n/a | yes | +| [is\_reservation\_active](#input\_is\_reservation\_active) | Whether the specified reservation is already created. | `bool` | `true` | no | +| [kubernetes\_labels](#input\_kubernetes\_labels) | Kubernetes labels to be applied to each node in the node group. Key-value pairs.
(The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) | `map(string)` | `null` | no | +| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_ssd\_count\_ephemeral\_storage](#input\_local\_ssd\_count\_ephemeral\_storage) | The number of local SSDs to attach to each node to back ephemeral storage.
Uses NVMe interfaces. Must be supported by `machine_type`.
When set to null, default value either is [set based on machine\_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value.
[See above](#local-ssd-storage) for more info. | `number` | `null` | no | +| [local\_ssd\_count\_nvme\_block](#input\_local\_ssd\_count\_nvme\_block) | The number of local SSDs to attach to each node to back block storage.
Uses NVMe interfaces. Must be supported by `machine_type`.
When set to null, default value either is [set based on machine\_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value.
[See above](#local-ssd-storage) for more info. | `number` | `null` | no | +| [machine\_type](#input\_machine\_type) | The name of a Google Compute Engine machine type. | `string` | `"c2-standard-60"` | no | +| [max\_pods\_per\_node](#input\_max\_pods\_per\_node) | The maximum number of pods per node in this node pool. This will force replacement. | `number` | `null` | no | +| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | +| [name](#input\_name) | The name of the node pool. If not set, automatically populated by machine type and module id (unique blueprint-wide) as suffix.
If setting manually, ensure a unique value across all gke-node-pools. | `string` | `null` | no | +| [num\_node\_pools](#input\_num\_node\_pools) | Number of node pools to create. This is same as num\_slices. | `number` | `1` | no | +| [num\_slices](#input\_num\_slices) | Number of TPUs slices to create. This is same as num\_node\_pools. | `number` | `1` | no | +| [placement\_policy](#input\_placement\_policy) | Group placement policy to use for the node pool's nodes. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. `tpu_topology` is the TPU placement topology for pod slice node pool.
It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. | 
object({
    type         = string
    name         = optional(string)
    tpu_topology = optional(string)
  })
| 
{
  "name": null,
  "tpu_topology": null,
  "type": null
}
| no | +| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | +| [reservation\_affinity](#input\_reservation\_affinity) | Reservation resource to consume. When targeting SPECIFIC\_RESERVATION, specific\_reservations needs be specified.
Even though specific\_reservations is a list, only one reservation is allowed by the NodePool API.
It is assumed that the specified reservation exists and has available capacity.
For a shared reservation, specify the project\_id as well in which it was created.
To create a reservation refer to https://cloud.google.com/compute/docs/instances/reservations-single-project and https://cloud.google.com/compute/docs/instances/reservations-shared | 
object({
    consume_reservation_type = string
    specific_reservations = optional(list(object({
      name    = string
      project = optional(string)
    })))
  })
| 
{
  "consume_reservation_type": "NO_RESERVATION",
  "specific_reservations": []
}
| no | +| [run\_workload\_script](#input\_run\_workload\_script) | Whether execute the script to create a sample workload and inject rxdm sidecar into workload. Currently, implemented for A3-Highgpu and A3-Megagpu only. | `bool` | `true` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: use service\_account\_email and scopes. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the node pool | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | +| [static\_node\_count](#input\_static\_node\_count) | The static number of nodes in the node pool. If set, autoscaling will be disabled. | `number` | `null` | no | +| [taints](#input\_taints) | Taints to be applied to the system node pool. | 
list(object({
    key    = string
    value  = any
    effect = string
  }))
| `[]` | no | +| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core. By setting threads\_per\_core
to 2, Simultaneous Multithreading (SMT) is enabled extending the total number
of virtual cores. For example, a machine of type c2-standard-60 will have 60
virtual cores with threads\_per\_core equal to 2. With threads\_per\_core equal
to 1 (SMT turned off), only the 30 physical cores will be available on the VM.

The default value of \"0\" will turn off SMT for supported machine types, and
will fall back to GCE defaults for unsupported machine types (t2d, shared-core
instances, or instances with less than 2 vCPU).

Disabling SMT can be more performant in many HPC workloads, therefore it is
disabled by default where compatible.

null = SMT configuration will use the GCE defaults for the machine type
0 = SMT will be disabled where compatible (default)
1 = SMT will always be disabled (will fail on incompatible machine types)
2 = SMT will always be enabled (will fail on incompatible machine types) | `number` | `0` | no | +| [timeout\_create](#input\_timeout\_create) | Timeout for creating a node pool | `string` | `null` | no | +| [timeout\_update](#input\_timeout\_update) | Timeout for updating a node pool | `string` | `null` | no | +| [total\_max\_nodes](#input\_total\_max\_nodes) | DEPRECATED: Use autoscaling\_total\_max\_nodes. | `number` | `null` | no | +| [total\_min\_nodes](#input\_total\_min\_nodes) | DEPRECATED: Use autoscaling\_total\_min\_nodes. | `number` | `null` | no | +| [upgrade\_settings](#input\_upgrade\_settings) | Defines node pool upgrade settings. It is highly recommended that you define all max\_surge and max\_unavailable.
If max\_surge is not specified, it would be set to a default value of 0.
If max\_unavailable is not specified, it would be set to a default value of 1. | 
object({
    strategy        = string
    max_surge       = optional(number)
    max_unavailable = optional(number)
  })
| 
{
  "max_surge": 0,
  "max_unavailable": 1,
  "strategy": "SURGE"
}
| no | +| [zones](#input\_zones) | A list of zones to be used. Zones must be in region of cluster. If null, cluster zones will be inherited. Note `zones` not `zone`; does not work with `zone` deployment variable. | `list(string)` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [allocatable\_cpu\_per\_node](#output\_allocatable\_cpu\_per\_node) | Number of CPUs available for scheduling pods on each node. | +| [allocatable\_gpu\_per\_node](#output\_allocatable\_gpu\_per\_node) | Number of GPUs available for scheduling pods on each node. | +| [has\_gpu](#output\_has\_gpu) | Boolean value indicating whether nodes in the pool are configured with GPUs. | +| [instructions](#output\_instructions) | Instructions for submitting the sample GPUDirect enabled job. | +| [node\_pool\_names](#output\_node\_pool\_names) | Names of the node pools. | +| [static\_gpu\_count](#output\_static\_gpu\_count) | Total number of GPUs in the node pool. Available only for static node pools. | +| [tolerations](#output\_tolerations) | Tolerations needed for a pod to be scheduled on this node pool. | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf new file mode 100644 index 0000000000..0c1c255255 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +## Required variables: +# local_ssd_count_ephemeral_storage +# local_ssd_count_nvme_block +# machine_type + +locals { + + local_ssd_machines = { + "a3-highgpu-8g" = { local_ssd_count_ephemeral_storage = 16, local_ssd_count_nvme_block = null }, + "a3-megagpu-8g" = { local_ssd_count_ephemeral_storage = 16, local_ssd_count_nvme_block = null }, + "a3-ultragpu-8g" = { local_ssd_count_ephemeral_storage = 32, local_ssd_count_nvme_block = null }, + "a4-highgpu-8g" = { local_ssd_count_ephemeral_storage = 32, local_ssd_count_nvme_block = null }, + } + + generated_local_ssd_config = lookup(local.local_ssd_machines, var.machine_type, { local_ssd_count_ephemeral_storage = null, local_ssd_count_nvme_block = null }) + + # Select in priority order: + # (1) var.local_ssd_count_ephemeral_storage and var.local_ssd_count_nvme_block if any is not null + # (2) local.local_ssd_machines if not empty + # (3) default to null value for both local_ssd_count_ephemeral_storage and local_ssd_count_nvme_block + local_ssd_config = (var.local_ssd_count_ephemeral_storage == null && var.local_ssd_count_nvme_block == null) ? local.generated_local_ssd_config : { local_ssd_count_ephemeral_storage = var.local_ssd_count_ephemeral_storage, local_ssd_count_nvme_block = var.local_ssd_count_nvme_block } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml new file mode 100644 index 0000000000..1106f63479 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml @@ -0,0 +1,50 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: my-sample-job +spec: + parallelism: 2 + completions: 2 + completionMode: Indexed + template: + spec: + containers: + - name: nccl-test + image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpx/nccl-plugin-gpudirecttcpx-dev:v3.1.9 + imagePullPolicy: Always + command: + - /bin/sh + - -c + - | + service ssh restart; + sleep infinity; + env: + - name: LD_LIBRARY_PATH + value: /usr/local/nvidia/lib64 + volumeMounts: + - name: config-volume + mountPath: /configs + resources: + limits: + nvidia.com/gpu: 8 + volumes: + - name: config-volume + configMap: + name: nccl-configmap + defaultMode: 0777 + restartPolicy: Never + backoffLimit: 0 diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml new file mode 100644 index 0000000000..90eea9c902 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml @@ -0,0 +1,70 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: Job +metadata: + name: my-sample-job +spec: + parallelism: 2 + completions: 2 + completionMode: Indexed + template: + spec: + hostname: host1 + subdomain: nccl-host-1 + containers: + - name: nccl-test + image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev:v1.0.8-1 + imagePullPolicy: Always + command: + - /bin/sh + - -c + - | + set -ex + chmod 755 /scripts/demo-run-nccl-test-tcpxo-via-mpi.sh + cat >/scripts/allgather.sh < 0: + container["env"].extend(env_vars) + container["volumeMounts"].extend(volume_mounts) + +if __name__ == "__main__": + main() diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py new file mode 100644 index 0000000000..db9fb3e7ff --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py @@ -0,0 +1,186 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml +import argparse +import os + +def main(): + parser = argparse.ArgumentParser(description="TCPXO Job Manifest Generator") + parser.add_argument("-f", "--file", required=True, help="Path to your job template YAML file") + parser.add_argument("-r", "--rxdm", required=True, help="RxDM version") + + args = parser.parse_args() + + # Get the YAML file from the user + if not args.file: + args.file = input("Please provide the path to your job template YAML file: ") + + # Get component versions from user + if not args.rxdm: + args.rxdm = input("Enter the RxDM version: ") + + # Load and modify the YAML + with open(args.file, "r") as file: + job_manifest = yaml.load(file, Loader=yaml.BaseLoader) + + # Update annotations + add_annotations(job_manifest) + + # Update volumes + add_volumes(job_manifest) + + # Update tolerations + add_tolerations(job_manifest) + + # Add tcpxo-daemon container + add_tcpxo_daemon_container(job_manifest, args.rxdm) + + # Update environment variables and volumeMounts for GPU containers + update_gpu_containers(job_manifest) + + # Generate the new YAML file + updated_job = str(yaml.dump(job_manifest, default_flow_style=False, width=1000, default_style="|", sort_keys=False)).replace("|-", "") + + new_file_name = args.file.replace(".yaml", "-tcpxo.yaml") + with open(new_file_name, "w", encoding="utf-8") as file: + file.write(updated_job) + + # Step 7: Provide instructions to the user + print("\nA new manifest has been generated and updated to have TCPXO enabled based on the provided workload") + print("It can be found in {path}".format(path=os.path.abspath(new_file_name))) + print("You can use the following commands to submit the sample job:") + print(" kubectl create -f {path}".format(path=os.path.abspath(new_file_name))) + +def add_annotations(job_manifest): + annotations = { + 'devices.gke.io/container.tcpxo-daemon':"""|+ +- path: /dev/nvidia0 +- path: /dev/nvidia1 +- path: /dev/nvidia2 +- path: /dev/nvidia3 +- path: /dev/nvidia4 +- path: /dev/nvidia5 +- path: /dev/nvidia6 +- path: /dev/nvidia7 +- path: /dev/nvidiactl +- path: /dev/nvidia-uvm +- path: /dev/dmabuf_import_helper""", + "networking.gke.io/default-interface": "eth0", + "networking.gke.io/interfaces": """| +[ + {"interfaceName":"eth0","network":"default"}, + {"interfaceName":"eth1","network":"vpc1"}, + {"interfaceName":"eth2","network":"vpc2"}, + {"interfaceName":"eth3","network":"vpc3"}, + {"interfaceName":"eth4","network":"vpc4"}, + {"interfaceName":"eth5","network":"vpc5"}, + {"interfaceName":"eth6","network":"vpc6"}, + {"interfaceName":"eth7","network":"vpc7"}, + {"interfaceName":"eth8","network":"vpc8"} +]""", + } + + # Create path if it doesn't exist + job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("metadata", {}) + + # Add/update annotations + pod_template_spec = job_manifest["spec"]["template"]["metadata"] + if "annotations" in pod_template_spec: + pod_template_spec["annotations"].update(annotations) + else: + pod_template_spec["annotations"] = annotations + +def add_tolerations(job_manifest): + tolerations = [ + {"key": "user-workload", "operator": "Equal", "value": """\"true\"""", "effect": "NoSchedule"}, + ] + + # Create path if it doesn't exist + job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) + + # Add tolerations + pod_spec = job_manifest["spec"]["template"]["spec"] + if "tolerations" in pod_spec: + pod_spec["tolerations"].extend(tolerations) + else: + pod_spec["tolerations"] = tolerations + +def add_volumes(job_manifest): + volumes = [ + {"name": "nvidia-install-dir-host", "hostPath": {"path": "/home/kubernetes/bin/nvidia"}}, + {"name": "sys", "hostPath": {"path": "/sys"}}, + {"name": "proc-sys", "hostPath": {"path": "/proc/sys"}}, + {"name": "aperture-devices", "hostPath": {"path": "/dev/aperture_devices"}}, + ] + + # Create path if it doesn't exist + job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) + + # Add volumes + pod_spec = job_manifest["spec"]["template"]["spec"] + if "volumes" in pod_spec: + pod_spec["volumes"].extend(volumes) + else: + pod_spec["volumes"] = volumes + + +def add_tcpxo_daemon_container(job_template, rxdm_version): + tcpxo_daemon_container = { + "name": "tcpxo-daemon", + "image": f"us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:{rxdm_version}", # Use provided RxDM version + "imagePullPolicy": "Always", + "command": ["/bin/sh", "-c"], + "args": [ + """| + set -ex + chmod 755 /fts/entrypoint_rxdm_container.sh + /fts/entrypoint_rxdm_container.sh --num_hops=2 --num_nics=8 --uid= --alsologtostderr""" + ], + "securityContext": { + "capabilities": {"add": ["NET_ADMIN", "NET_BIND_SERVICE"]} + }, + "volumeMounts": [ + {"name": "nvidia-install-dir-host", "mountPath": "/usr/local/nvidia"}, + {"name": "sys", "mountPath": "/hostsysfs"}, + {"name": "proc-sys", "mountPath": "/hostprocsysfs"}, + ], + "env": [{"name": "LD_LIBRARY_PATH", "value": "/usr/local/nvidia/lib64"}], + } + + # Create path if it doesn't exist + job_template.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) + + # Add container + pod_spec = job_template["spec"]["template"]["spec"] + pod_spec.setdefault("containers", []).insert(0, tcpxo_daemon_container) + +def update_gpu_containers(job_manifest): + env_vars = [ + {"name": "LD_LIBRARY_PATH", "value": "/usr/local/nvidia/lib64"}, + {"name": "NCCL_FASTRAK_LLCM_DEVICE_DIRECTORY", "value": "/dev/aperture_devices"}, + ] + volume_mounts = [{"name": "aperture-devices", "mountPath": "/dev/aperture_devices"}] + + pod_spec = job_manifest.get("spec", {}).get("template", {}).get("spec", {}) + for container in pod_spec.get("containers", []): + # Create path if it doesn't exist + container.setdefault("env", []) + container.setdefault("volumeMounts", []) + if int(container.get("resources", {}).get("limits", {}).get("nvidia.com/gpu", 0)) > 0: + container["env"].extend(env_vars) + container["volumeMounts"].extend(volume_mounts) + +if __name__ == "__main__": + main() diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf new file mode 100644 index 0000000000..002141f800 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf @@ -0,0 +1,87 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +# Enable GPUDirect for A3 and A3Mega VMs, this involve multiple kubectl steps to integrate with the created cluster +# 1. Install NCCL plugin daemonset +# 2. Install NRI plugin daemonset +# 3. Update provided workload to inject rxdm sidecar and other required annotation, volume etc. +locals { + workload_path_tcpx = "${path.module}/gpu-direct-workload/sample-tcpx-workload-job.yaml" + workload_path_tcpxo = "${path.module}/gpu-direct-workload/sample-tcpxo-workload-job.yaml" + + gpu_direct_settings = { + "a3-highgpu-8g" = { + # Manifest to be installed for enabling TCPX on a3-highgpu-8g machines + gpu_direct_manifests = [ + "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/gpudirect-tcpx/nccl-tcpx-installer.yaml", # nccl_plugin v3.1.9 for tcpx + "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/gpudirect-tcpx/nccl-config.yaml", # nccl_configmap + "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/nri_device_injector/nri-device-injector.yaml", # nri_plugin + ] + updated_workload_path = replace(local.workload_path_tcpx, ".yaml", "-tcpx.yaml") + rxdm_version = "v2.0.12" # matching nccl-tcpx-installer version v3.1.9 + min_additional_networks = 4 + major_minor_version_acceptable_map = { + "1.27" = "1.27.7-gke.1121000" + "1.28" = "1.28.8-gke.1095000" + "1.29" = "1.29.3-gke.1093000" + "1.30" = "1.30.2-gke.1023000" + } + } + "a3-megagpu-8g" = { + # Manifest to be installed for enabling TCPXO on a3-megagpu-8g machines + gpu_direct_manifests = [ + "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/39308db7574925ea3c14f9113fcf87f70a6fcc26/gpudirect-tcpxo/nccl-tcpxo-installer.yaml", # nccl_plugin v1.0.8-1 for tcpxo + "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/39308db7574925ea3c14f9113fcf87f70a6fcc26/nri_device_injector/nri-device-injector.yaml", # nri_plugin + ] + updated_workload_path = replace(local.workload_path_tcpxo, ".yaml", "-tcpxo.yaml") + rxdm_version = "v1.0.14" # matching nccl-tcpxo-installer version v1.0.8-1 + min_additional_networks = 8 + major_minor_version_acceptable_map = { + "1.28" = "1.28.9-gke.1250000" + "1.29" = "1.29.4-gke.1542000" + "1.30" = "1.30.4-gke.1129000" + "1.31" = "1.31.1-gke.2008000" + "1.32" = "1.32.2-gke.1489001" + } + } + } + + min_additional_networks = try(local.gpu_direct_settings[var.machine_type].min_additional_networks, 0) + + gke_version_regex = "(\\d+\\.\\d+)\\.(\\d+)-gke\\.(\\d+)" # GKE version format: 1.X.Y-gke.Z , regex output: ["1.X" , "Y", "Z"] + + gke_version_parts = regex(local.gke_version_regex, var.gke_version) + gke_version_major = local.gke_version_parts[0] + + major_minor_version_acceptable_map = try(local.gpu_direct_setting[var.machine_type].major_minor_version_acceptable_map, null) + minor_version_acceptable = try(contains(keys(local.major_minor_version_acceptable_map), local.gke_version_major), false) ? local.major_minor_version_acceptable_map[local.gke_version_major] : "1.0.0-gke.0" + minor_version_acceptable_parts = regex(local.gke_version_regex, local.minor_version_acceptable) + gke_gpudirect_compatible = local.gke_version_parts[1] > local.minor_version_acceptable_parts[1] || (local.gke_version_parts[1] == local.minor_version_acceptable_parts[1] && local.gke_version_parts[2] >= local.minor_version_acceptable_parts[2]) +} + +check "gpu_direct_check_multi_vpc" { + assert { + condition = length(var.additional_networks) >= local.min_additional_networks + error_message = "To achieve optimal performance for ${var.machine_type} machine, at least ${local.min_additional_networks} additional vpc is recommended. You could configure it in the blueprint through modules/network/multivpc with network_count set as ${local.min_additional_networks}" + } +} + +check "gke_version_requirements" { + assert { + condition = local.gke_gpudirect_compatible + error_message = "GPUDirect is not supported on GKE version ${var.gke_version} for ${var.machine_type} machine. For supported version details visit https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#requirements" + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf new file mode 100644 index 0000000000..1ddc7ba8c3 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +data "google_compute_machine_types" "machine_info" { + for_each = var.zones == null ? toset([]) : toset(var.zones) + + project = var.project_id + zone = each.key + filter = "name = \"${var.machine_type}\"" +} + +locals { + valid_machine_info = { + for zone, data in data.google_compute_machine_types.machine_info : + zone => data.machine_types if length(data.machine_types) > 0 + } + + guest_cpus = try(local.valid_machine_info[0].guest_cpus, 0) +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/main.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/main.tf new file mode 100644 index 0000000000..857fe4dbc8 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/main.tf @@ -0,0 +1,440 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "gke-node-pool", ghpc_role = "compute" }) +} + +locals { + upgrade_settings = { + strategy = var.upgrade_settings.strategy + max_surge = coalesce(var.upgrade_settings.max_surge, 0) + max_unavailable = coalesce(var.upgrade_settings.max_unavailable, 1) + } +} + +module "gpu" { + source = "../../internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + has_gpu = length(local.guest_accelerator) > 0 + allocatable_gpu_per_node = local.has_gpu ? max(local.guest_accelerator[*].count...) : -1 + is_static_node_pool_with_gpus = var.static_node_count != null && local.allocatable_gpu_per_node != -1 + static_gpu_count = local.is_static_node_pool_with_gpus ? var.static_node_count * local.allocatable_gpu_per_node : 0 + gpu_taint = local.has_gpu ? [{ + key = "nvidia.com/gpu" + value = "present" + effect = "NO_SCHEDULE" + }] : [] + + autoscale_set = var.autoscaling_total_min_nodes != 0 || var.autoscaling_total_max_nodes != 1000 + static_node_set = var.static_node_count != null + initial_node_set = try(var.initial_node_count > 0, false) + + module_unique_id = replace(lower(var.internal_ghpc_module_id), "/[^a-z0-9\\-]/", "") +} + + +locals { + cluster_id_parts = split("/", var.cluster_id) + cluster_name = local.cluster_id_parts[5] + cluster_location = local.cluster_id_parts[3] +} + + +data "google_container_cluster" "gke_cluster" { + name = local.cluster_name + location = local.cluster_location +} + +resource "google_container_node_pool" "node_pool" { + provider = google + + count = max(var.num_node_pools, var.num_slices) + + name = (max(var.num_node_pools, var.num_slices) == 1) ? coalesce(var.name, join("-", [var.machine_type, local.module_unique_id])) : join("-", [coalesce(var.name, join("-", [var.machine_type, local.module_unique_id])), count.index]) + cluster = var.cluster_id + node_locations = var.zones + + node_count = var.static_node_count + dynamic "autoscaling" { + for_each = local.static_node_set ? [] : [1] + content { + total_min_node_count = var.autoscaling_total_min_nodes + total_max_node_count = var.autoscaling_total_max_nodes + location_policy = "ANY" + } + } + + initial_node_count = var.initial_node_count + + max_pods_per_node = var.max_pods_per_node + + management { + auto_repair = var.auto_repair + auto_upgrade = var.auto_upgrade + } + + upgrade_settings { + strategy = local.upgrade_settings.strategy + max_surge = local.upgrade_settings.max_surge + max_unavailable = local.upgrade_settings.max_unavailable + } + + dynamic "placement_policy" { + for_each = var.placement_policy.type != null ? [1] : [] + content { + type = var.placement_policy.type + policy_name = var.placement_policy.name + tpu_topology = var.placement_policy.tpu_topology + } + } + + dynamic "queued_provisioning" { + for_each = var.enable_queued_provisioning ? [1] : [] + content { + enabled = true + } + } + + node_config { + disk_size_gb = var.disk_size_gb + disk_type = var.disk_type + resource_labels = local.labels + labels = var.kubernetes_labels + service_account = var.service_account_email + oauth_scopes = var.service_account_scopes + machine_type = var.machine_type + spot = var.spot + image_type = var.image_type + flex_start = var.enable_flex_start + max_run_duration = var.max_run_duration != null ? "${var.max_run_duration}s" : null + + dynamic "guest_accelerator" { + for_each = local.guest_accelerator + iterator = ga + content { + type = coalesce(ga.value.type, try(local.generated_guest_accelerator[0].type, "")) + count = coalesce(try(ga.value.count, 0) > 0 ? ga.value.count : try(local.generated_guest_accelerator[0].count, "0")) + + gpu_partition_size = try(ga.value.gpu_partition_size, null) + + dynamic "gpu_driver_installation_config" { + # in case user did not specify guest_accelerator settings, we need a try to default to [] + for_each = try([ga.value.gpu_driver_installation_config], [{ gpu_driver_version = "DEFAULT" }]) + iterator = gdic + content { + gpu_driver_version = gdic.value.gpu_driver_version + } + } + + dynamic "gpu_sharing_config" { + for_each = try(ga.value.gpu_sharing_config == null, true) ? [] : [ga.value.gpu_sharing_config] + iterator = gsc + content { + gpu_sharing_strategy = gsc.value.gpu_sharing_strategy + max_shared_clients_per_gpu = gsc.value.max_shared_clients_per_gpu + } + } + } + } + + dynamic "taint" { + for_each = concat(var.taints, local.gpu_taint) + content { + key = taint.value.key + value = taint.value.value + effect = taint.value.effect + } + } + + dynamic "ephemeral_storage_local_ssd_config" { + for_each = local.local_ssd_config.local_ssd_count_ephemeral_storage != null ? [1] : [] + content { + local_ssd_count = local.local_ssd_config.local_ssd_count_ephemeral_storage + } + } + + dynamic "local_nvme_ssd_block_config" { + for_each = local.local_ssd_config.local_ssd_count_nvme_block != null ? [1] : [] + content { + local_ssd_count = local.local_ssd_config.local_ssd_count_nvme_block + } + } + + shielded_instance_config { + enable_secure_boot = var.enable_secure_boot + enable_integrity_monitoring = true + } + + dynamic "gcfs_config" { + for_each = var.enable_gcfs ? [1] : [] + content { + enabled = true + } + } + + gvnic { + enabled = var.image_type == "COS_CONTAINERD" + } + + dynamic "advanced_machine_features" { + for_each = local.set_threads_per_core ? [1] : [] + content { + threads_per_core = local.threads_per_core # relies on threads_per_core_calc.tf + } + } + + # Implied by Workload Identity + workload_metadata_config { + mode = "GKE_METADATA" + } + # Implied by workload identity. + metadata = { + "disable-legacy-endpoints" = "true" + } + + linux_node_config { + sysctls = { + "net.ipv4.tcp_rmem" = "4096 87380 16777216" + "net.ipv4.tcp_wmem" = "4096 16384 16777216" + } + } + + reservation_affinity { + consume_reservation_type = var.reservation_affinity.consume_reservation_type + key = local.is_valid_reservation ? local.reservation_resource_api_label : null + values = local.is_valid_reservation ? (var.is_reservation_active ? local.active_reservation_values : local.default_reservation_values) : null + } + + dynamic "host_maintenance_policy" { + for_each = var.host_maintenance_interval != "" ? [1] : [] + content { + maintenance_interval = var.host_maintenance_interval + } + } + } + + network_config { + dynamic "additional_node_network_configs" { + for_each = var.additional_networks + + content { + network = additional_node_network_configs.value.network + subnetwork = additional_node_network_configs.value.subnetwork + } + } + + enable_private_nodes = var.enable_private_nodes + } + + timeouts { + create = var.timeout_create + update = var.timeout_update + } + + lifecycle { + ignore_changes = [ + node_config[0].labels, + initial_node_count, + # Ignore local/ephemeral ssd configs as they are tied to machine types. + node_config[0].ephemeral_storage_local_ssd_config, + node_config[0].local_nvme_ssd_block_config, + ] + precondition { + condition = (var.max_pods_per_node == null) || (data.google_container_cluster.gke_cluster.networking_mode == "VPC_NATIVE") + error_message = "max_pods_per_node does not work on `routes-based` clusters, that don't have IP Aliasing enabled." + } + precondition { + condition = !local.static_node_set || !local.autoscale_set + error_message = "static_node_count cannot be set with either autoscaling_total_min_nodes or autoscaling_total_max_nodes." + } + precondition { + condition = !local.static_node_set || !local.initial_node_set + error_message = "initial_node_count cannot be set with static_node_count." + } + precondition { + condition = !local.initial_node_set || (coalesce(var.initial_node_count, 0) >= var.autoscaling_total_min_nodes && coalesce(var.initial_node_count, 0) <= var.autoscaling_total_max_nodes) + error_message = "initial_node_count must be between autoscaling_total_min_nodes and autoscaling_total_max_nodes included." + } + precondition { + condition = !(coalesce(local.local_ssd_config.local_ssd_count_ephemeral_storage, 0) > 0 && coalesce(local.local_ssd_config.local_ssd_count_nvme_block, 0) > 0) + error_message = "Only one of local_ssd_count_ephemeral_storage or local_ssd_count_nvme_block can be set to a non-zero value." + } + precondition { + condition = ( + (var.reservation_affinity.consume_reservation_type != "SPECIFIC_RESERVATION" && local.input_specific_reservations_count == 0) || + (var.reservation_affinity.consume_reservation_type == "SPECIFIC_RESERVATION" && local.input_specific_reservations_count == 1) + ) + error_message = <<-EOT + When using NO_RESERVATION or ANY_RESERVATION as the `consume_reservation_type`, `specific_reservations` cannot be set. + On the other hand, with SPECIFIC_RESERVATION you must set `specific_reservations`. + EOT + } + precondition { + condition = ( + (local.input_specific_reservations_count == 0) || + ((length(local.verified_specific_reservations) == 1 || !var.is_reservation_active) && + length(local.specific_reservation_requirement_violations) == 0) + ) + error_message = <<-EOT + Check if your reservation is configured correctly: + - A reservation with the name must exist in the specified project and one of the specified zones + + - Its consumption type must be "specific" + %{for property in local.specific_reservation_requirement_violations} + - ${local.specific_reservation_requirement_violation_messages[property]} + %{endfor} + EOT + } + precondition { + condition = ( + (local.input_specific_reservations_count == 0) || + (local.input_specific_reservations_count == 1 && length(local.input_reservation_suffixes) == 0) || + (local.input_specific_reservations_count == 1 && length(local.input_reservation_suffixes) > 0 && try(local.input_reservation_projects[0], var.project_id) == var.project_id) + ) + error_message = "Shared extended reservations are not supported by GKE." + } + precondition { + condition = contains(["SURGE"], local.upgrade_settings.strategy) + error_message = "Only SURGE strategy is supported" + } + precondition { + condition = local.upgrade_settings.max_unavailable >= 0 + error_message = "max_unavailable should be set to 0 or greater" + } + precondition { + condition = local.upgrade_settings.max_surge >= 0 + error_message = "max_surge should be set to 0 or greater" + } + precondition { + condition = local.upgrade_settings.max_unavailable > 0 || local.upgrade_settings.max_surge > 0 + error_message = "At least one of max_unavailable or max_surge must greater than 0" + } + precondition { + condition = var.placement_policy.type != "COMPACT" || (var.zones != null ? (length(var.zones) == 1) : false) + error_message = "Compact placement is only available for node pools operating in a single zone." + } + precondition { + condition = var.placement_policy.type != "COMPACT" || local.upgrade_settings.strategy != "BLUE_GREEN" + error_message = "Compact placement is not supported with blue-green upgrades." + } + precondition { + condition = !(var.enable_queued_provisioning == true && var.placement_policy.type == "COMPACT") + error_message = "placement_policy cannot be COMPACT when enable_queued_provisioning is true." + } + precondition { + condition = !(var.enable_queued_provisioning == true && var.reservation_affinity.consume_reservation_type != "NO_RESERVATION") + error_message = "reservation_affinity should be NO_RESERVATION when enable_queued_provisioning is true." + } + precondition { + condition = !(var.enable_queued_provisioning == true && var.autoscaling_total_min_nodes != 0) + error_message = "autoscaling_total_min_nodes should be 0 when enable_queued_provisioning is true." + } + precondition { + condition = !(var.num_node_pools > 1 && var.num_slices > 1) + error_message = "num_node_pools is for CPUs and GPUS, and num_slices is for TPUs. Both cannot be set at the same time to create a group of identical nodepools / slices." + } + precondition { + condition = !(var.num_node_pools == 0 && var.num_slices == 0) + error_message = "Either num_node_pools (for CPUs and GPUS) or num_slices (for TPUs) should be set to a positive integer value." + } + precondition { + condition = !(var.num_node_pools < 0 || var.num_slices < 0) + error_message = "Negative integer value of num_node_pools or num_slices is not valid. Please use a positive integer value to set num_node_pools for CPUs and GPUS, and num_slices for TPUs." + } + precondition { + condition = var.enable_flex_start == true ? (var.auto_repair == false) : true + error_message = "enable_flex_start needs node auto_repair set to false." + } + precondition { + condition = var.enable_flex_start == true ? (var.static_node_count == null) : true + error_message = "enable_flex_start does not work with static_node_count. static_node_count should be set to null." + } + precondition { + condition = var.enable_flex_start == true ? (var.reservation_affinity.consume_reservation_type == "NO_RESERVATION") : true + error_message = "enable_flex_start only works with reservation_affinity consume_reservation_type NO_RESERVATION." + } + precondition { + condition = var.enable_flex_start == true ? (var.spot == false) : true + error_message = "Both enable_flex_start and spot consumption option cannot be set to true at the same time." + } + } +} + +locals { + supported_machine_types_for_install_dependencies = ["a3-highgpu-8g", "a3-megagpu-8g"] +} + +resource "null_resource" "install_dependencies" { + count = var.run_workload_script && contains(local.supported_machine_types_for_install_dependencies, var.machine_type) ? 1 : 0 + provisioner "local-exec" { + command = "pip3 install pyyaml" + } +} + +locals { + gpu_direct_setting = lookup(local.gpu_direct_settings, var.machine_type, { gpu_direct_manifests = [], updated_workload_path = "", rxdm_version = "" }) +} + +# execute script to inject rxdm sidecar into workload to enable tcpx for a3-highgpu-8g VM workload +resource "null_resource" "enable_tcpx_in_workload" { + count = var.run_workload_script && var.machine_type == "a3-highgpu-8g" ? 1 : 0 + triggers = { + always_run = timestamp() + } + provisioner "local-exec" { + command = "python3 ${path.module}/gpu-direct-workload/scripts/enable-tcpx-in-workload.py --file ${local.workload_path_tcpx} --rxdm ${local.gpu_direct_setting.rxdm_version}" + } + + depends_on = [null_resource.install_dependencies] +} + +# execute script to inject rxdm sidecar into workload to enable tcpxo for a3-megagpu-8g VM workload +resource "null_resource" "enable_tcpxo_in_workload" { + count = var.run_workload_script && var.machine_type == "a3-megagpu-8g" ? 1 : 0 + triggers = { + always_run = timestamp() + } + provisioner "local-exec" { + command = "python3 ${path.module}/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py --file ${local.workload_path_tcpxo} --rxdm ${local.gpu_direct_setting.rxdm_version}" + } + + depends_on = [null_resource.install_dependencies] +} + +# apply manifest to enable tcpx +module "kubectl_apply" { + source = "../../management/kubectl-apply" + + cluster_id = var.cluster_id + project_id = var.project_id + + apply_manifests = flatten([ + for manifest in local.gpu_direct_setting.gpu_direct_manifests : [ + { + source = manifest + } + ] + ]) +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/metadata.yaml new file mode 100644 index 0000000000..e980d595a2 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com +ghpc: + inject_module_id: internal_ghpc_module_id diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/outputs.tf new file mode 100644 index 0000000000..aa88c7d5bc --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/outputs.tf @@ -0,0 +1,112 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "node_pool_names" { + description = "Names of the node pools." + value = google_container_node_pool.node_pool[*].name +} + +locals { + # Shared core machines only have 1 cpu allocatable, even if they have 2 cpu capacity + vcpu = local.machine_shared_core ? 1 : local.guest_cpus + useable_cpu = local.set_threads_per_core ? local.threads_per_core * local.vcpu / 2 : local.vcpu + + # allocatable resource definition: https://cloud.google.com/kubernetes-engine/docs/concepts/plan-node-sizes#cpu_reservations + second_core = local.useable_cpu > 1 ? 1 : 0 + third_fourth_core = local.useable_cpu == 3 ? 1 : local.useable_cpu > 3 ? 2 : 0 + cores_above_four = local.useable_cpu > 4 ? local.useable_cpu - 4 : 0 + + allocatable_cpu = 0.94 + (0.99 * local.second_core) + (0.995 * local.third_fourth_core) + (0.9975 * local.cores_above_four) +} + +output "allocatable_cpu_per_node" { + description = "Number of CPUs available for scheduling pods on each node." + value = local.allocatable_cpu +} + +output "has_gpu" { + description = "Boolean value indicating whether nodes in the pool are configured with GPUs." + value = local.has_gpu +} + +output "allocatable_gpu_per_node" { + description = "Number of GPUs available for scheduling pods on each node." + value = local.allocatable_gpu_per_node +} + +output "static_gpu_count" { + description = "Total number of GPUs in the node pool. Available only for static node pools." + value = local.static_gpu_count +} + +locals { + translate_toleration = { + PREFER_NO_SCHEDULE = "PreferNoSchedule" + NO_SCHEDULE = "NoSchedule" + NO_EXECUTE = "NoExecute" + } + taints = google_container_node_pool.node_pool[0].node_config[0].taint + tolerations = [for taint in local.taints : { + key = taint.key + operator = "Equal" + value = taint.value + effect = lookup(local.translate_toleration, taint.effect, null) + }] +} + +output "tolerations" { + description = "Tolerations needed for a pod to be scheduled on this node pool." + value = local.tolerations +} + +locals { + gpu_direct_enabled = var.machine_type == "a3-highgpu-8g" || var.machine_type == "a3-megagpu-8g" + script_path = { + a3-highgpu-8g = "enable-tcpx-in-workload.py", + a3-megagpu-8g = "enable-tcpxo-in-workload.py" + } + nccl_path = var.machine_type == "a3-highgpu-8g" ? "configs" : "scripts" + gpu_direct_instruction = <<-EOT + Since you are using ${var.machine_type} machine type that has GPUDirect support, your nodepool had been configured with the required plugins. + To fully utilize GPUDirect you will need to add some components into your workload manifest. Details below: + + A sample GKE job that has GPUDirect enabled and NCCL test included has been generated locally at: + ${abspath(local.gpu_direct_setting.updated_workload_path)} + + You can use the following commands to submit the sample job: + kubectl create -f ${abspath(local.gpu_direct_setting.updated_workload_path)} + After submitting the sample job, you can validate the GPU performance by initiating NCCL test included in the sample workload: + NCCL test can be initiated from any one of the sample job Pods and coordinate with the peer Pods: + export POD_NAME=$(kubectl get pods -l job-name=my-sample-job -o go-template='{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' | head -n 1) + export PEER_POD_IPS=$(kubectl get pods -l job-name=my-sample-job -o go-template='{{range .items}}{{.status.podIP}}{{" "}}{{end}}') + kubectl exec --stdin --tty --container=nccl-test $POD_NAME -- /${local.nccl_path}/allgather.sh $PEER_POD_IPS + + If you would like to enable GPUDirect for your own workload, please follow the below steps: + export WORKLOAD_PATH=<> + python3 ${abspath("${path.module}/gpu-direct-workload/scripts/${lookup(local.script_path, var.machine_type, "")}")} --file $WORKLOAD_PATH --rxdm ${local.gpu_direct_setting.rxdm_version} + **WARNING** + The "--rxdm" version is tied to the nccl-tcpx/o-installer that had been deployed to your cluster, changing it to other value might have impact on performance + **WARNING** + + Or you can also follow our GPUDirect user guide to update your workload + https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#add-gpudirect-manifests + EOT +} + +output "instructions" { + description = "Instructions for submitting the sample GPUDirect enabled job." + value = local.gpu_direct_enabled ? local.gpu_direct_instruction : null +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf new file mode 100644 index 0000000000..359e7c5aa8 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf @@ -0,0 +1,99 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +# Split the input into three different lists where the details of a given reservation are at the same index across these lists. +locals { + # Specific block of an extended reservation can be targeted with exr-one/reservationBlocks/exr-one-block-1 + # Data source needs to be queried with the reservation name only. So, we extract the reservation name + input_reservation_names = [for r in try(var.reservation_affinity.specific_reservations, []) : split("/", r.name)[0]] + input_reservation_projects = [for r in try(var.reservation_affinity.specific_reservations, []) : coalesce(r.project, var.project_id)] + # We, also, remember the suffix "/reservationBlocks/exr-one-block-1" for use elsewhere afterwards + input_reservation_suffixes = [for r in try(var.reservation_affinity.specific_reservations, []) : substr(r.name, length(split("/", r.name)[0]), -1)] +} + +data "google_compute_reservation" "specific_reservations" { + for_each = ( + local.input_specific_reservations_count == 0 ? + {} : + { + for pair in flatten([ + for zone in try(var.zones, []) : [ + for i, reservation_name in try(local.input_reservation_names, []) : { + key : "${local.input_reservation_projects[i]}/${zone}/${reservation_name}" + zone : zone + reservation_name : reservation_name + project : local.input_reservation_projects[i] + } + ] + ]) : + pair.key => pair + } + ) + name = each.value.reservation_name + zone = each.value.zone + project = each.value.project +} + +locals { + generated_guest_accelerator = module.gpu.machine_type_guest_accelerator + reservation_resource_api_label = "compute.googleapis.com/reservation-name" + input_specific_reservations_count = try(length(var.reservation_affinity.specific_reservations), 0) + + # Filter specific reservations + verified_specific_reservations = [for k, v in data.google_compute_reservation.specific_reservations : v if(v.specific_reservation != null && v.specific_reservation_required == true)] + + # Build two maps to be used to compare the VM properties between reservations and the node pool + # Skip this for TPUs + reservation_vm_properties = local.has_gpu ? ([for reservation in local.verified_specific_reservations : { + "machine_type" : try(reservation.specific_reservation[0].instance_properties[0].machine_type, "") + "guest_accelerators" : { for acc in try(reservation.specific_reservation[0].instance_properties[0].guest_accelerators, []) : acc.accelerator_type => acc.accelerator_count } + }]) : [] + nodepool_vm_properties = { + "machine_type" : var.machine_type + "guest_accelerators" : { for acc in try(local.guest_accelerator, []) : coalesce(acc.type, try(local.generated_guest_accelerator[0].type, "")) => coalesce(acc.count, try(local.generated_guest_accelerator[0].count, 0)) } + } + + # Compare two maps by counting the keys that mismatch. + # Know that in map comparison the order of keys does not matter. That is {NVME: x, SCSI: y} and {SCSI: y, NVME: x} are equal + # As of this writing, there is only one reservation supported by the Node Pool API. So, directly accessing it from the list + specific_reservation_requirement_violations = length(local.reservation_vm_properties) == 0 ? [] : [for k, v in local.nodepool_vm_properties : k if v != local.reservation_vm_properties[0][k]] + + specific_reservation_requirement_violation_messages = { + "machine_type" : <<-EOT + The reservation has "${try(local.reservation_vm_properties[0].machine_type, "")}" machine type and the node pool has "${local.nodepool_vm_properties.machine_type}". Check the relevant node pool setting: "machine_type" + EOT + "guest_accelerators" : <<-EOT + The reservation has ${jsonencode(try(local.reservation_vm_properties[0].guest_accelerators, {}))} accelerators and the node pool has ${jsonencode(try(local.nodepool_vm_properties.guest_accelerators, {}))}. Check the relevant node pool setting: "guest_accelerator". When unspecified, for the machine_type=${var.machine_type}, the default is guest_accelerator=${jsonencode(try(local.generated_guest_accelerator, [{}]))}. + EOT + } +} + +locals { + # Check if reservation is valid, that is, if it exists, there should be only 1 verified specific reservation or the reservation doesn't exist + is_valid_reservation = length(local.verified_specific_reservations) == 1 || !var.is_reservation_active + + # Build the list of reservation names when var.is_reservation_active is true + active_reservation_values = [ + for i, r in local.verified_specific_reservations : + length(local.input_reservation_suffixes[i]) > 0 ? + format("%s%s", r.name, local.input_reservation_suffixes[i]) : + "projects/${r.project}/reservations/${r.name}" + ] + + # Define a default reservation value if no specific reservations are present + specific_reservation_name = length(local.input_reservation_names) > 0 ? local.input_reservation_names[0] : "" + default_reservation_values = ["projects/${var.project_id}/reservations/${local.specific_reservation_name}"] +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf new file mode 100644 index 0000000000..e582db33da --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf @@ -0,0 +1,42 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# This file is meant to be reused by multiple modules. +# "description": Allows for 'threads_per_core=0: SMT will be disabled where compatible (default)' + +# "inputs": +# var.machine_type: Machine type for the instance being evaluated. +# var.threads_per_core : Sets the number of threads per physical core, where 0 +# has behavior described in description. + +# "outputs": +# local.set_threads_per_core: bool that tells if threads per core should be set, +# to be used with a dynamic block. +# local.threads_per_core: actual threads_per_core to be used. + +locals { + machine_vals = split("-", var.machine_type) + machine_family = local.machine_vals[0] + machine_shared_core = length(local.machine_vals) <= 2 + machine_vcpus = try(parseint(local.machine_vals[2], 10), 1) + + smt_capable_family = !contains(["t2d", "t2a"], local.machine_family) + smt_capable_vcpu = local.machine_vcpus >= 2 + + smt_capable = local.smt_capable_family && local.smt_capable_vcpu && !local.machine_shared_core + set_threads_per_core = var.threads_per_core != null && (var.threads_per_core == 0 && local.smt_capable || try(var.threads_per_core >= 1, false)) + threads_per_core = var.threads_per_core == 2 ? 2 : 1 +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/variables.tf new file mode 100644 index 0000000000..cfd1dcd6b0 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/variables.tf @@ -0,0 +1,472 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "The project ID to host the cluster in." + type = string +} + +variable "cluster_id" { + description = "projects/{{project}}/locations/{{location}}/clusters/{{cluster}}" + type = string +} + +variable "zones" { + description = "A list of zones to be used. Zones must be in region of cluster. If null, cluster zones will be inherited. Note `zones` not `zone`; does not work with `zone` deployment variable." + type = list(string) + default = null +} + +variable "name" { + description = <<-EOD + The name of the node pool. If not set, automatically populated by machine type and module id (unique blueprint-wide) as suffix. + If setting manually, ensure a unique value across all gke-node-pools. + EOD + type = string + default = null +} + +variable "internal_ghpc_module_id" { + description = "DO NOT SET THIS MANUALLY. Automatically populates with module id (unique blueprint-wide)." + type = string +} + +variable "machine_type" { + description = "The name of a Google Compute Engine machine type." + type = string + default = "c2-standard-60" +} + +variable "disk_size_gb" { + description = "Size of disk for each node." + type = number + default = 100 +} + +variable "disk_type" { + description = "Disk type for each node." + type = string + default = null +} + +variable "enable_gcfs" { + description = "Enable the Google Container Filesystem (GCFS). See [restrictions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#gcfs_config)." + type = bool + default = false +} + +variable "enable_secure_boot" { + description = "Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info." + type = bool + default = true +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = optional(string) + count = optional(number, 0) + gpu_driver_installation_config = optional(object({ + gpu_driver_version = string + }), { gpu_driver_version = "DEFAULT" }) + gpu_partition_size = optional(string) + gpu_sharing_config = optional(object({ + gpu_sharing_strategy = string + max_shared_clients_per_gpu = number + })) + })) + default = [] + nullable = false + + validation { + condition = alltrue([for ga in var.guest_accelerator : ga.count != null]) + error_message = "var.guest_accelerator[*].count cannot be null" + } + + validation { + condition = alltrue([for ga in var.guest_accelerator : ga.count >= 0]) + error_message = "var.guest_accelerator[*].count must never be negative" + } + + validation { + condition = alltrue([for ga in var.guest_accelerator : ga.gpu_driver_installation_config != null]) + error_message = "var.guest_accelerator[*].gpu_driver_installation_config must not be null; leave unset to enable GKE to select default GPU driver installation" + } +} + +variable "image_type" { + description = "The default image type used by NAP once a new node pool is being created. Use either COS_CONTAINERD or UBUNTU_CONTAINERD." + type = string + default = "COS_CONTAINERD" +} + +variable "local_ssd_count_ephemeral_storage" { + description = <<-EOT + The number of local SSDs to attach to each node to back ephemeral storage. + Uses NVMe interfaces. Must be supported by `machine_type`. + When set to null, default value either is [set based on machine_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value. + [See above](#local-ssd-storage) for more info. + EOT + type = number + default = null +} + +variable "local_ssd_count_nvme_block" { + description = <<-EOT + The number of local SSDs to attach to each node to back block storage. + Uses NVMe interfaces. Must be supported by `machine_type`. + When set to null, default value either is [set based on machine_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value. + [See above](#local-ssd-storage) for more info. + + EOT + type = number + default = null +} + +variable "autoscaling_total_min_nodes" { + description = "Total minimum number of nodes in the NodePool." + type = number + default = 0 +} + +variable "autoscaling_total_max_nodes" { + description = "Total maximum number of nodes in the NodePool." + type = number + default = 1000 +} + +variable "static_node_count" { + description = "The static number of nodes in the node pool. If set, autoscaling will be disabled." + type = number + default = null +} + +variable "is_reservation_active" { + description = "Whether the specified reservation is already created." + type = bool + default = true +} + +variable "auto_repair" { + description = "Whether the nodes will be automatically repaired." + type = bool + default = true +} + +variable "auto_upgrade" { + description = "Whether the nodes will be automatically upgraded." + type = bool + default = false +} + +variable "threads_per_core" { + description = <<-EOT + Sets the number of threads per physical core. By setting threads_per_core + to 2, Simultaneous Multithreading (SMT) is enabled extending the total number + of virtual cores. For example, a machine of type c2-standard-60 will have 60 + virtual cores with threads_per_core equal to 2. With threads_per_core equal + to 1 (SMT turned off), only the 30 physical cores will be available on the VM. + + The default value of \"0\" will turn off SMT for supported machine types, and + will fall back to GCE defaults for unsupported machine types (t2d, shared-core + instances, or instances with less than 2 vCPU). + + Disabling SMT can be more performant in many HPC workloads, therefore it is + disabled by default where compatible. + + null = SMT configuration will use the GCE defaults for the machine type + 0 = SMT will be disabled where compatible (default) + 1 = SMT will always be disabled (will fail on incompatible machine types) + 2 = SMT will always be enabled (will fail on incompatible machine types) + EOT + type = number + default = 0 + + validation { + condition = var.threads_per_core == null || try(var.threads_per_core >= 0, false) && try(var.threads_per_core <= 2, false) + error_message = "Allowed values for threads_per_core are \"null\", \"0\", \"1\", \"2\"." + } +} + +variable "spot" { + description = "Provision VMs using discounted Spot pricing, allowing for preemption" + type = bool + default = false +} + +# tflint-ignore: terraform_unused_declarations +variable "compact_placement" { + description = "DEPRECATED: Use `placement_policy`" + type = bool + default = null + validation { + condition = var.compact_placement == null + error_message = "`compact_placement` is deprecated. Use `placement_policy` instead" + } +} + +variable "placement_policy" { + description = <<-EOT + Group placement policy to use for the node pool's nodes. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. `tpu_topology` is the TPU placement topology for pod slice node pool. + It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement. + Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. + EOT + + type = object({ + type = string + name = optional(string) + tpu_topology = optional(string) + }) + default = { + type = null + name = null + tpu_topology = null + } + validation { + condition = var.placement_policy.type == null || try(contains(["COMPACT"], var.placement_policy.type), false) + error_message = "`COMPACT` is the only supported value for `placement_policy.type`." + } +} + +variable "service_account_email" { + description = "Service account e-mail address to use with the node pool" + type = string + default = null +} + +variable "service_account_scopes" { + description = "Scopes to to use with the node pool." + type = set(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +variable "taints" { + description = "Taints to be applied to the system node pool." + type = list(object({ + key = string + value = any + effect = string + })) + default = [] +} + +variable "labels" { + description = "GCE resource labels to be applied to resources. Key-value pairs." + type = map(string) +} + +variable "kubernetes_labels" { + description = <<-EOT + Kubernetes labels to be applied to each node in the node group. Key-value pairs. + (The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) + EOT + type = map(string) + default = null +} + +variable "timeout_create" { + description = "Timeout for creating a node pool" + type = string + default = null +} + +variable "timeout_update" { + description = "Timeout for updating a node pool" + type = string + default = null +} + +# Deprecated + +# tflint-ignore: terraform_unused_declarations +variable "total_min_nodes" { + description = "DEPRECATED: Use autoscaling_total_min_nodes." + type = number + default = null + validation { + condition = var.total_min_nodes == null + error_message = "total_min_nodes was renamed to autoscaling_total_min_nodes and is deprecated; use autoscaling_total_min_nodes" + } +} + +# tflint-ignore: terraform_unused_declarations +variable "total_max_nodes" { + description = "DEPRECATED: Use autoscaling_total_max_nodes." + type = number + default = null + validation { + condition = var.total_max_nodes == null + error_message = "total_max_nodes was renamed to autoscaling_total_max_nodes and is deprecated; use autoscaling_total_max_nodes" + } +} + +# tflint-ignore: terraform_unused_declarations +variable "service_account" { + description = "DEPRECATED: use service_account_email and scopes." + type = object({ + email = string, + scopes = set(string) + }) + default = null + validation { + condition = var.service_account == null + error_message = "service_account is deprecated and replaced with service_account_email and scopes." + } +} + +variable "additional_networks" { + description = "Additional network interface details for GKE, if any. Providing additional networks adds additional node networks to the node pool" + default = [] + type = list(object({ + network = string + subnetwork = string + subnetwork_project = string + network_ip = string + nic_type = string + stack_type = string + queue_count = number + access_config = list(object({ + nat_ip = string + network_tier = string + })) + ipv6_access_config = list(object({ + network_tier = string + })) + alias_ip_range = list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })) + })) + nullable = false +} + +variable "reservation_affinity" { + description = <<-EOT + Reservation resource to consume. When targeting SPECIFIC_RESERVATION, specific_reservations needs be specified. + Even though specific_reservations is a list, only one reservation is allowed by the NodePool API. + It is assumed that the specified reservation exists and has available capacity. + For a shared reservation, specify the project_id as well in which it was created. + To create a reservation refer to https://cloud.google.com/compute/docs/instances/reservations-single-project and https://cloud.google.com/compute/docs/instances/reservations-shared + EOT + type = object({ + consume_reservation_type = string + specific_reservations = optional(list(object({ + name = string + project = optional(string) + }))) + }) + default = { + consume_reservation_type = "NO_RESERVATION" + specific_reservations = [] + } + validation { + condition = contains(["NO_RESERVATION", "ANY_RESERVATION", "SPECIFIC_RESERVATION"], var.reservation_affinity.consume_reservation_type) + error_message = "Accepted values are: {NO_RESERVATION, ANY_RESERVATION, SPECIFIC_RESERVATION}" + } +} + +variable "host_maintenance_interval" { + description = "Specifies the frequency of planned maintenance events." + type = string + default = "" + nullable = false + validation { + condition = contains(["", "PERIODIC", "AS_NEEDED"], var.host_maintenance_interval) + error_message = "Invalid host_maintenance_interval value. Must be PERIODIC, AS_NEEDED or the empty string" + } +} + +variable "initial_node_count" { + description = "The initial number of nodes for the pool. In regional clusters, this is the number of nodes per zone. Changing this setting after node pool creation will not make any effect. It cannot be set with static_node_count and must be set to a value between autoscaling_total_min_nodes and autoscaling_total_max_nodes." + type = number + default = null +} + +variable "gke_version" { + description = "GKE version" + type = string +} + +variable "max_pods_per_node" { + description = "The maximum number of pods per node in this node pool. This will force replacement." + type = number + default = null +} + +variable "upgrade_settings" { + description = <<-EOT + Defines node pool upgrade settings. It is highly recommended that you define all max_surge and max_unavailable. + If max_surge is not specified, it would be set to a default value of 0. + If max_unavailable is not specified, it would be set to a default value of 1. + EOT + type = object({ + strategy = string + max_surge = optional(number) + max_unavailable = optional(number) + }) + default = { + strategy = "SURGE" + max_surge = 0 + max_unavailable = 1 + } +} + +variable "run_workload_script" { + description = "Whether execute the script to create a sample workload and inject rxdm sidecar into workload. Currently, implemented for A3-Highgpu and A3-Megagpu only." + type = bool + default = true +} + +variable "enable_queued_provisioning" { + description = "If true, enables Dynamic Workload Scheduler and adds the cloud.google.com/gke-queued taint to the node pool." + type = bool + default = false +} + +variable "enable_flex_start" { + description = <<-EOT + If true, start the node pool with Flex Start provisioning model. + To learn more about flex-start mode, please refer to + https://cloud.google.com/kubernetes-engine/docs/how-to/dws-flex-start-training and + https://cloud.google.com/kubernetes-engine/docs/how-to/provisioningrequest + EOT + type = bool + default = false +} + +variable "max_run_duration" { + description = "The duration (in whole seconds) of the instance. Instance will run and be terminated after then." + type = number + default = null +} + +variable "enable_private_nodes" { + description = "Whether nodes have internal IP addresses only." + type = bool + default = false +} + +variable "num_node_pools" { + description = "Number of node pools to create. This is same as num_slices." + type = number + default = 1 +} + +variable "num_slices" { + description = "Number of TPUs slices to create. This is same as num_node_pools." + type = number + default = 1 +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/versions.tf new file mode 100644 index 0000000000..26df685e61 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/versions.tf @@ -0,0 +1,34 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.5" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.41" + } + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.57.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.45.0" + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/README.md b/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/README.md new file mode 100644 index 0000000000..d91a923310 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/README.md @@ -0,0 +1,82 @@ +## Description + +This modules create a [resource policy for compute engines](https://cloud.google.com/compute/docs/instances/placement-policies-overview). This policy can be passed to a gke-node-pool module to apply the policy on the node-pool's nodes. + +Note: By default, you can't apply compact placement policies with a max distance value to A3 VMs. To request access to this feature, contact your [Technical Account Manager (TAM)](https://cloud.google.com/tam) or the [Sales team](https://cloud.google.com/contact). + +### Example + +The following example creates a group placement resource policy and applies it to a gke-node-pool. + +```yaml + - id: group_placement_1 + source: modules/compute/resource-policy + settings: + name: gp-np-1 + group_placement_max_distance: 2 + + - id: node_pool_1 + source: modules/compute/gke-node-pool + use: [group_placement_1] + settings: + machine_type: e2-standard-8 + outputs: [instructions] +``` + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google-beta](#requirement\_google-beta) | >= 6.29.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google-beta](#provider\_google-beta) | >= 6.29.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_resource_policy.policy](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_resource_policy) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [group\_placement\_max\_distance](#input\_group\_placement\_max\_distance) | The max distance for group placement policy to use for the node pool's nodes. If set it will add a compact group placement policy.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. | `number` | `0` | no | +| [name](#input\_name) | The resource policy's name. | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | The project ID for the resource policy. | `string` | n/a | yes | +| [region](#input\_region) | The region for the the resource policy. | `string` | n/a | yes | +| [workload\_policy](#input\_workload\_policy) | Describes the workload policy | 
object({
    type                  = optional(string, null)
    max_topology_distance = optional(string, null)
    accelerator_topology  = optional(string, null)
  })
| 
{
  "accelerator_topology": null,
  "max_topology_distance": null,
  "type": null
}
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| [placement\_policy](#output\_placement\_policy) | Group placement policy to use for placing VMs or GKE nodes placement. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy.
It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/main.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/main.tf new file mode 100644 index 0000000000..906424ca7c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/main.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +locals { + name = "${var.name}-${random_id.resource_name_suffix.hex}" +} + +resource "google_compute_resource_policy" "policy" { + name = local.name + region = var.region + project = var.project_id + provider = google-beta + + dynamic "workload_policy" { + for_each = var.workload_policy.type != null ? [1] : [] + + content { + type = var.workload_policy.type + max_topology_distance = var.workload_policy.max_topology_distance + accelerator_topology = var.workload_policy.accelerator_topology + } + } + + dynamic "group_placement_policy" { + for_each = var.group_placement_max_distance > 0 ? [1] : [] + + content { + collocation = "COLLOCATED" + max_distance = var.group_placement_max_distance + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/outputs.tf new file mode 100644 index 0000000000..b4b2a4e1b3 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/outputs.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "placement_policy" { + description = <<-EOT + Group placement policy to use for placing VMs or GKE nodes placement. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. + It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement. + Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. + EOT + + value = { + type = (var.group_placement_max_distance > 0 || var.workload_policy.type != null) ? "COMPACT" : null + name = (var.group_placement_max_distance > 0 || var.workload_policy.type != null) ? local.name : null + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/variables.tf new file mode 100644 index 0000000000..e70ecca8a0 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/variables.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID for the resource policy." + type = string +} + +variable "region" { + description = "The region for the the resource policy." + type = string +} + +variable "name" { + description = "The resource policy's name." + type = string +} + +variable "group_placement_max_distance" { + description = <<-EOT + The max distance for group placement policy to use for the node pool's nodes. If set it will add a compact group placement policy. + Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. + EOT + + type = number + default = 0 +} + +variable "workload_policy" { + description = "Describes the workload policy" + type = object({ + type = optional(string, null) + max_topology_distance = optional(string, null) + accelerator_topology = optional(string, null) + }) + default = { + type = null + max_topology_distance = null + accelerator_topology = null + } + nullable = false +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/versions.tf new file mode 100644 index 0000000000..f235fbade3 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/versions.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.29.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:resource-policy/v1.37.2" + } + + required_version = ">= 1.3" +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/README.md b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/README.md new file mode 100644 index 0000000000..c2535ee03e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/README.md @@ -0,0 +1,258 @@ +## Description + +This module creates one or more +[compute VM instances](https://cloud.google.com/compute/docs/instances). + +### Example + +```yaml +- id: compute + source: modules/compute/vm-instance + use: [network1] + settings: + instance_count: 8 + name_prefix: compute + machine_type: c2-standard-60 +``` + +This creates a cluster of 8 compute VMs that are: + +* named `compute-[0-7]` +* on the network defined by the `network1` module +* of type c2-standard-60 + +> **_NOTE:_** Simultaneous Multithreading (SMT) is deactivated by default +> (threads_per_core=1), which means only the physical cores are visible on the +> VM. With SMT disabled, a machine of type c2-standard-60 will only have the 30 +> physical cores visible. To change this, set `threads_per_core=2` under +> settings. + +### VPC Networks + +There are two methods for adding network connectivity to the `vm-instance` +module. The first is shown in the example above, where a `vpc` module or +`pre-existing-vpc` module is used by the `vm-instance` module. When this +happens, the `network_self_link` and `subnetwork_self_link` outputs from the +network are provided as input to the `vm-instance` and a network interface is +defined based on that. This can also be done updating the `network_self_link` and +`subnetwork_self_link` settings directly. + +The alternative option can be used when more than one network needs to be added +to the `vm-instance` or further customization is needed beyond what is provided +via other variables. For this option, the `network_interfaces` variable can be +used to set up one or more network interfaces on the VM instance. The format is +consistent with the terraform `google_compute_instance` `network_interface` +block, and more information can be found in the +[terraform docs][network-interface-tf]. + +> **_NOTE:_** When supplying the `network_interfaces` variable, networks +> associated with the `vm-instance` via use will be ignored in favor of the +> networks added in `network_interfaces`. In addition, `bandwidth_tier` and +> `disable_public_ips` will not apply to networks defined in +> `network_interfaces`. + +[network-interface-tf]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface + +### SSH key metadata + +This module will ignore all changes to the `ssh-keys` metadata field that are +typically set by [external Google Cloud tools that automate SSH access][gcpssh] +when not using OS Login. For example, clicking on the Google Cloud Console SSH +button next to VMs in the VM Instances list will temporarily modify VM metadata +to include a dynamically-generated SSH public key. + +[gcpssh]: https://cloud.google.com/compute/docs/connect/add-ssh-keys#metadata + +### Placement + +The `placement_policy` variable can be used to control where your VM instances +are physically located relative to each other within a zone. See the official +placement [guide][guide-link] and [api][api-link] documentation. + +[guide-link]: https://cloud.google.com/compute/docs/instances/define-instance-placement +[api-link]: https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement + +Use the following settings for compact placement: + +```yaml + ... + settings: + instance_count: 4 + machine_type: c2-standard-60 + placement_policy: + collocation: "COLLOCATED" +``` + +By default the above placement policy will always result in the most compact set +of VMs available. If you would like that provisioning failed if some level of +compactness is not obtainable, you can enforce this with the [`max_distance` +setting](https://cloud.google.com/compute/docs/instances/use-compact-placement-policies): + +```yaml + ... + settings: + instance_count: 4 + machine_type: c2-standard-60 + placement_policy: + collocation: "COLLOCATED" + max_distance: 1 +``` + +Use the following settings for spread placement: + +```yaml + ... + settings: + instance_count: 4 + machine_type: n2-standard-4 + placement_policy: + availability_domain_count: 2 +``` + +When `vm_count` is not set, as shown in the examples above, then the VMs will be +added to the placement policy incrementally. This is the **recommended way** to +use placement policies. + +If `vm_count` is specified then VMs will stay in pending state until the +specified number of VMs are created. See the warning below if using this field. + +> [!WARNING] +> When creating a compact placement using `vm_count` with more than 10 VMs, you +> must add `-parallelism=` argument on apply. For example if you have 15 VMs +> in a placement group: `terraform apply -parallelism=15`. This is because +> terraform self limits to 10 parallel requests by default but the create +> instance requests will not succeed until all VMs in the placement group have +> been requested, forming a deadlock. + +### GPU Support + +More information on GPU support in `vm-instance` and other Cluster Toolkit modules +can be found at [docs/gpu-support.md](../../../docs/gpu-support.md) + +## Lifecycle + +The `vm-instance` module will be replaced when the `instance_image` variable is +changed and `terraform apply` is run on the deployment group folder or +`gcluster deploy` is run. However, it will not be automatically replaced if a new +image is created in a family. + +To selectively replace the vm-instance(s), consider running terraform +`apply -replace` such as: + +> See https://developer.hashicorp.com/terraform/cli/commands/plan#replace-address for precise syntax terraform apply -replace=ADDRESS + +```shell +terraform state list +# search for the module ID and resource +terraform apply -replace="address" +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [google](#requirement\_google) | >= 4.73.0 | +| [google-beta](#requirement\_google-beta) | >= 6.13.0 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.73.0 | +| [google-beta](#provider\_google-beta) | >= 6.13.0 | +| [null](#provider\_null) | >= 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gpu](#module\_gpu) | ../../internal/gpu-definition | n/a | +| [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | ../../scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_instance.compute_vm](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance) | resource | +| [google-beta_google_compute_resource_policy.placement_policy](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_resource_policy) | resource | +| [google_compute_address.compute_ip](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | +| [google_compute_disk.additional_disks](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [google_compute_disk.boot_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | +| [null_resource.image](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.replace_vm_trigger_from_placement](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [add\_deployment\_name\_before\_prefix](#input\_add\_deployment\_name\_before\_prefix) | If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments.
See `name_prefix` for further details on resource naming behavior. | `bool` | `false` | no | +| [additional\_persistent\_disks](#input\_additional\_persistent\_disks) | Configurations of additional disks to be included on the partition nodes. | 
object({
    count = optional(number, 0)
    type  = optional(string, "pd-balanced")
    size  = optional(number, 200)
  })
| `{}` | no | +| [allocate\_ip](#input\_allocate\_ip) | If not null, allocate IPs with the given configuration. See details at
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address | 
object({
    address_type = optional(string, "INTERNAL")
    purpose      = optional(string),
    network_tier = optional(string),
    ip_version   = optional(string, "IPV4"),
  })
| `null` | no | +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [auto\_delete\_boot\_disk](#input\_auto\_delete\_boot\_disk) | Controls if boot disk should be auto-deleted when instance is deleted. | `bool` | `true` | no | +| [automatic\_restart](#input\_automatic\_restart) | Specifies if the instance should be restarted if it was terminated by Compute Engine (not a user). | `bool` | `null` | no | +| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image that supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"not_enabled"` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment, will optionally be used name resources according to `name_prefix` | `string` | n/a | yes | +| [disable\_public\_ips](#input\_disable\_public\_ips) | If set to true, instances will not have public IPs | `bool` | `false` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for instances. | `number` | `200` | no | +| [disk\_type](#input\_disk\_type) | Disk type for instances. | `string` | `"pd-standard"` | no | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | +| [instance\_count](#input\_instance\_count) | Number of instances | `number` | `1` | no | +| [instance\_image](#input\_instance\_image) | Instance Image | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_ssd\_count](#input\_local\_ssd\_count) | The number of local SSDs to attach to each VM. See https://cloud.google.com/compute/docs/disks/local-ssd. | `number` | `0` | no | +| [local\_ssd\_interface](#input\_local\_ssd\_interface) | Interface to be used with local SSDs. Can be either 'NVME' or 'SCSI'. No effect unless `local_ssd_count` is also set. | `string` | `"NVME"` | no | +| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation | `string` | `"c2-standard-60"` | no | +| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | +| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | +| [name\_prefix](#input\_name\_prefix) | An optional name for all VM and disk resources.
If not supplied, `deployment_name` will be used.
When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set,
then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". | `string` | `null` | no | +| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces. The options match that of the terraform
network\_interface block of google\_compute\_instance. For descriptions of the
subfields or more information see the documentation:
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface

**\_NOTE:\_** If `network_interfaces` are set, `network_self_link` and
`subnetwork_self_link` will be ignored, even if they are provided through
the `use` field. `bandwidth_tier` and `disable_public_ips` also do not apply
to network interfaces defined in this variable.

Subfields:
network (string, required if subnetwork is not supplied)
subnetwork (string, required if network is not supplied)
subnetwork\_project (string, optional)
network\_ip (string, optional)
nic\_type (string, optional, choose from ["GVNIC", "VIRTIO\_NET", "MRDMA", "IRDMA"])
stack\_type (string, optional, choose from ["IPV4\_ONLY", "IPV4\_IPV6"])
queue\_count (number, optional)
access\_config (object, optional)
ipv6\_access\_config (object, optional)
alias\_ip\_range (list(object), optional) | 
list(object({
    network            = string,
    subnetwork         = string,
    subnetwork_project = string,
    network_ip         = string,
    nic_type           = string,
    stack_type         = string,
    queue_count        = number,
    access_config = list(object({
      nat_ip                 = string,
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    ipv6_access_config = list(object({
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    alias_ip_range = list(object({
      ip_cidr_range         = string,
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the VM. Can use "default" for the default network. | `string` | `null` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE` | `string` | `null` | no | +| [placement\_policy](#input\_placement\_policy) | Control where your VM instances are physically located relative to each other within a zone.
See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_resource_policy#nested_group_placement_policy | `any` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [provisioning\_model](#input\_provisioning\_model) | Provisioning model for cloud instance. | `string` | `null` | no | +| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | +| [reservation\_name](#input\_reservation\_name) | Name of the reservation to use for VM resources, should be in one of the following formats:
- projects/PROJECT\_ID/reservations/RESERVATION\_NAME
- RESERVATION\_NAME

Must be a "SPECIFIC\_RESERVATION"
Set to empty string if using no reservation or automatically-consumed reservations | `string` | `""` | no | +| [service\_account](#input\_service\_account) | DEPRECATED - Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the node pool | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [spot](#input\_spot) | DEPRECATED - Use `provisioning_model` instead. | `bool` | `null` | no | +| [startup\_script](#input\_startup\_script) | Startup script used on the instance | `string` | `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the VM. | `string` | `null` | no | +| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | +| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core. By setting threads\_per\_core
to 2, Simultaneous Multithreading (SMT) is enabled extending the total number
of virtual cores. For example, a machine of type c2-standard-60 will have 60
virtual cores with threads\_per\_core equal to 2. With threads\_per\_core equal
to 1 (SMT turned off), only the 30 physical cores will be available on the VM.

The default value of \"0\" will turn off SMT for supported machine types, and
will fall back to GCE defaults for unsupported machine types (t2d, shared-core
instances, or instances with less than 2 vCPU).

Disabling SMT can be more performant in many HPC workloads, therefore it is
disabled by default where compatible.

null = SMT configuration will use the GCE defaults for the machine type
0 = SMT will be disabled where compatible (default)
1 = SMT will always be disabled (will fail on incompatible machine types)
2 = SMT will always be enabled (will fail on incompatible machine types) | `number` | `0` | no | +| [zone](#input\_zone) | Compute Platform zone | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [external\_ip](#output\_external\_ip) | External IP of the instances (if enabled) | +| [instructions](#output\_instructions) | Instructions on how to SSH into the created VM. Commands may fail depending on VM configuration and IAM permissions. | +| [internal\_ip](#output\_internal\_ip) | Internal IP of the instances | +| [name](#output\_name) | Names of instances created | +| [self\_link](#output\_self\_link) | The tuple URIs of the created instances | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/compute_image.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/compute_image.tf new file mode 100644 index 0000000000..7a7fe02307 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/compute_image.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_compute_image" "compute_image" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = try(var.instance_image.project, null) + + lifecycle { + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/main.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/main.tf new file mode 100644 index 0000000000..9618ea9179 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/main.tf @@ -0,0 +1,349 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "vm-instance", ghpc_role = "compute" }) +} + +module "gpu" { + source = "../../internal/gpu-definition" + + machine_type = var.machine_type + guest_accelerator = var.guest_accelerator +} + +locals { + guest_accelerator = module.gpu.guest_accelerator + + native_fstype = [] + startup_script = local.startup_from_network_storage != null ? ( + { startup-script = local.startup_from_network_storage }) : {} + network_storage = var.network_storage != null ? ( + { network_storage = jsonencode(var.network_storage) }) : {} + + prefix_optional_deployment_name = var.name_prefix != null ? var.name_prefix : var.deployment_name + prefix_always_deployment_name = var.name_prefix != null ? "${var.deployment_name}-${var.name_prefix}" : var.deployment_name + resource_prefix = var.add_deployment_name_before_prefix ? local.prefix_always_deployment_name : local.prefix_optional_deployment_name + + enable_gvnic = var.bandwidth_tier != "not_enabled" + enable_tier_1 = var.bandwidth_tier == "tier_1_enabled" + + provisioning_model = var.provisioning_model + + spot = var.provisioning_model == "SPOT" + + # compact_placement : true when placement policy is provided and collocation set; false if unset + compact_placement = try(var.placement_policy.collocation, null) != null + + gpu_attached = contains(["a2", "g2"], local.machine_family) || length(local.guest_accelerator) > 0 + + # both of these must be false if either compact placement or preemptible/spot instances are used + # automatic restart is tolerant of GPUs while on host maintenance is not + automatic_restart_default = local.compact_placement || local.spot ? false : null + on_host_maintenance_default = local.compact_placement || local.spot || local.gpu_attached ? "TERMINATE" : "MIGRATE" + + automatic_restart = ( + var.automatic_restart != null + ? var.automatic_restart + : local.automatic_restart_default + ) + + on_host_maintenance = ( + var.on_host_maintenance != null + ? var.on_host_maintenance + : local.on_host_maintenance_default + ) + + oslogin_api_values = { + "DISABLE" = "FALSE" + "ENABLE" = "TRUE" + } + enable_oslogin = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } + + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + # Network Interfaces + # Support for `use` input and base network parameters like `network_self_link` and `subnetwork_self_link` + empty_access_config = { + nat_ip = null, + public_ptr_domain_name = null, + network_tier = null + } + default_network_interface = { + network = var.network_self_link + subnetwork = var.subnetwork_self_link + subnetwork_project = null # will populate from subnetwork_self_link + network_ip = null + nic_type = local.enable_gvnic ? "GVNIC" : null + stack_type = null + queue_count = null + access_config = var.disable_public_ips ? [] : [local.empty_access_config] + ipv6_access_config = [] + alias_ip_range = [] + } + network_interfaces = coalescelist(var.network_interfaces, [local.default_network_interface]) + network_interfaces_with_ips = var.allocate_ip == null ? local.network_interfaces : [ + for i, interface in local.network_interfaces : + merge(interface, { + network_ip = google_compute_address.compute_ip[i].address + }) + ] +} + +resource "null_resource" "image" { + triggers = { + name = try(var.instance_image.name, null), + family = try(var.instance_image.family, null), + project = try(var.instance_image.project, null) + } +} + +resource "google_compute_disk" "boot_disk" { + project = var.project_id + + count = var.instance_count + + name = "${local.resource_prefix}-boot-disk-${count.index}" + image = data.google_compute_image.compute_image.self_link + type = var.disk_type + size = var.disk_size_gb + labels = local.labels + zone = var.zone + + lifecycle { + replace_triggered_by = [null_resource.image] + + ignore_changes = [ + image + ] + } +} + +resource "google_compute_disk" "additional_disks" { + project = var.project_id + + count = var.instance_count * var.additional_persistent_disks.count + + # NB: this resource array must be sliced accounting for var.instance_count + name = "${local.resource_prefix}-disk-${count.index}" + type = var.additional_persistent_disks.type + size = var.additional_persistent_disks.size + labels = local.labels + zone = var.zone +} + +resource "google_compute_resource_policy" "placement_policy" { + project = var.project_id + provider = google-beta + + count = var.placement_policy != null ? 1 : 0 + name = "${local.resource_prefix}-vm-instance-placement" + group_placement_policy { + vm_count = try(var.placement_policy.vm_count, null) + availability_domain_count = try(var.placement_policy.availability_domain_count, null) + collocation = try(var.placement_policy.collocation, null) + max_distance = try(var.placement_policy.max_distance, null) + } +} + +resource "null_resource" "replace_vm_trigger_from_placement" { + triggers = { + vm_count = try(tostring(var.placement_policy.vm_count), "") + availability_domain_count = try(tostring(var.placement_policy.availability_domain_count), "") + max_distance = try(tostring(var.placement_policy.max_distance), "") + collocation = try(var.placement_policy.collocation, "") + } +} + +resource "google_compute_address" "compute_ip" { + project = var.project_id + + count = var.allocate_ip != null ? length(local.network_interfaces) : 0 + + name = "${local.resource_prefix}-${count.index}" + + address = local.network_interfaces[count.index].network_ip + region = var.region + network = can(coalesce(local.network_interfaces[count.index].subnetwork)) ? null : local.network_interfaces[count.index].network + subnetwork = local.network_interfaces[count.index].subnetwork + address_type = var.allocate_ip.address_type + purpose = var.allocate_ip.purpose + network_tier = var.allocate_ip.network_tier + ip_version = var.allocate_ip.ip_version +} + +resource "google_compute_instance" "compute_vm" { + project = var.project_id + provider = google-beta + + count = var.instance_count + + depends_on = [var.network_self_link, var.network_storage] + + name = "${local.resource_prefix}-${count.index}" + min_cpu_platform = var.min_cpu_platform + machine_type = var.machine_type + zone = var.zone + + resource_policies = google_compute_resource_policy.placement_policy[*].self_link + + tags = var.tags + labels = local.labels + + boot_disk { + source = google_compute_disk.boot_disk[count.index].self_link + device_name = google_compute_disk.boot_disk[count.index].name + auto_delete = var.auto_delete_boot_disk + } + + dynamic "attached_disk" { + for_each = slice( + google_compute_disk.additional_disks, + var.additional_persistent_disks.count * count.index, + var.additional_persistent_disks.count * count.index + var.additional_persistent_disks.count, + ) + + content { + source = attached_disk.value.self_link + device_name = "additional-disk-${attached_disk.key}" + mode = "READ_WRITE" + } + } + + dynamic "scratch_disk" { + for_each = range(var.local_ssd_count) + content { + interface = var.local_ssd_interface + } + } + + dynamic "network_interface" { + for_each = local.network_interfaces_with_ips + + content { + network = network_interface.value.network + subnetwork = network_interface.value.subnetwork + subnetwork_project = network_interface.value.subnetwork_project + network_ip = network_interface.value.network_ip + nic_type = network_interface.value.nic_type + stack_type = network_interface.value.stack_type + queue_count = network_interface.value.queue_count + dynamic "access_config" { + for_each = network_interface.value.access_config + content { + nat_ip = access_config.value.nat_ip + public_ptr_domain_name = access_config.value.public_ptr_domain_name + network_tier = access_config.value.network_tier + } + } + dynamic "ipv6_access_config" { + for_each = network_interface.value.ipv6_access_config + content { + public_ptr_domain_name = ipv6_access_config.value.public_ptr_domain_name + network_tier = ipv6_access_config.value.network_tier + } + } + dynamic "alias_ip_range" { + for_each = network_interface.value.alias_ip_range + content { + ip_cidr_range = alias_ip_range.value.ip_cidr_range + subnetwork_range_name = alias_ip_range.value.subnetwork_range_name + } + } + } + } + + network_performance_config { + total_egress_bandwidth_tier = local.enable_tier_1 ? "TIER_1" : "DEFAULT" + } + + service_account { + email = var.service_account_email + scopes = var.service_account_scopes + } + + dynamic "guest_accelerator" { + for_each = local.guest_accelerator + content { + count = guest_accelerator.value.count + type = guest_accelerator.value.type + } + } + + scheduling { + on_host_maintenance = local.on_host_maintenance + automatic_restart = local.automatic_restart + preemptible = local.spot + provisioning_model = local.provisioning_model + } + + dynamic "advanced_machine_features" { + for_each = local.set_threads_per_core ? [1] : [] + content { + threads_per_core = local.threads_per_core # relies on threads_per_core_calc.tf + } + } + + dynamic "reservation_affinity" { + for_each = var.reservation_name == "" ? [] : [1] + content { + type = "SPECIFIC_RESERVATION" + specific_reservation { + key = "compute.googleapis.com/reservation-name" + values = [var.reservation_name] + } + } + } + + metadata = merge( + local.network_storage, + local.startup_script, + local.enable_oslogin, + local.disable_automatic_updates_metadata, + var.metadata + ) + + lifecycle { + ignore_changes = [ + metadata["ssh-keys"], + ] + + replace_triggered_by = [ + null_resource.replace_vm_trigger_from_placement + ] + + precondition { + condition = (length(var.network_interfaces) == 0) != (var.network_self_link == null && var.subnetwork_self_link == null) + error_message = "Exactly one of network_interfaces or network_self_link/subnetwork_self_link must be specified." + } + precondition { + condition = alltrue([for interface in var.network_interfaces : interface.network_ip == null]) || var.instance_count == 1 + error_message = <<-EOT + The network_ip cannot be statically set on vm-instance when the VM instance_count is greater than 1. + Either set the network_ip to null to allow it to be set dynamically for all instances, or create modules for each VM instance with its own network interface. + EOT + } + precondition { + condition = !contains([ + "c3-:pd-standard", + "h3-:pd-standard", + "h3-:pd-ssd", + ], "${substr(var.machine_type, 0, 3)}:${var.disk_type}") + error_message = "A disk_type=${var.disk_type} cannot be used with machine_type=${var.machine_type}." + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/outputs.tf new file mode 100644 index 0000000000..eab8cb56bd --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/outputs.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "name" { + description = "Names of instances created" + value = google_compute_instance.compute_vm[*].name +} + +output "self_link" { + description = "The tuple URIs of the created instances" + value = google_compute_instance.compute_vm[*].self_link +} + +output "external_ip" { + description = "External IP of the instances (if enabled)" + value = try(google_compute_instance.compute_vm[*].network_interface[0].access_config[0].nat_ip, []) +} + +output "internal_ip" { + description = "Internal IP of the instances" + value = google_compute_instance.compute_vm[*].network_interface[0].network_ip +} + +locals { + first_instance_link = try(google_compute_instance.compute_vm[0].self_link, "no-instance") + ssh_instructions = <<-EOT + Use the following commands to SSH into the first VM created: + gcloud compute ssh ${local.first_instance_link} --project ${var.project_id} + If not accessible from the public internet, use an SSH tunnel through IAP: + gcloud compute ssh ${local.first_instance_link} --tunnel-through-iap --project ${var.project_id} + EOT +} + +output "instructions" { + description = "Instructions on how to SSH into the created VM. Commands may fail depending on VM configuration and IAM permissions." + value = var.instance_count > 0 ? local.ssh_instructions : "No instances were created." +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf new file mode 100644 index 0000000000..02bc58e4f7 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# This file is meant to be reused by multiple modules. +# "inputs": +# local.native_fstype : list of file systems that are supported automatically, but looking at the metadata. +# var.network_storage : to be passed into metadata somewhere else (not here) +# var.startup_script : to be changed into a more complete file system with all the fs runners + +# "outputs": +# local.startup_from_network_storage : A full startup script with all the runners that are not supported +# natively and were included in the network_storage structure + +locals { + startup_script_network_storage = [ + for ns in var.network_storage : + ns if !contains(local.native_fstype, ns.fs_type) + ] + # Pull out runners to include in startup script + storage_client_install_runners = [ + for ns in local.startup_script_network_storage : + ns.client_install_runner if ns.client_install_runner != null + ] + mount_runners = [ + for ns in local.startup_script_network_storage : + ns.mount_runner if ns.mount_runner != null + ] + + startup_script_runner = [{ + content = var.startup_script != null ? var.startup_script : "echo 'No user provided startup script.'" + destination = "passed_startup_script.sh" + type = "shell" + }] + + full_runner_list = concat( + local.storage_client_install_runners, + local.mount_runners, + local.startup_script_runner + ) + + startup_from_network_storage = module.netstorage_startup_script.startup_script +} + +module "netstorage_startup_script" { + source = "../../scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.full_runner_list +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf new file mode 100644 index 0000000000..e582db33da --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf @@ -0,0 +1,42 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# This file is meant to be reused by multiple modules. +# "description": Allows for 'threads_per_core=0: SMT will be disabled where compatible (default)' + +# "inputs": +# var.machine_type: Machine type for the instance being evaluated. +# var.threads_per_core : Sets the number of threads per physical core, where 0 +# has behavior described in description. + +# "outputs": +# local.set_threads_per_core: bool that tells if threads per core should be set, +# to be used with a dynamic block. +# local.threads_per_core: actual threads_per_core to be used. + +locals { + machine_vals = split("-", var.machine_type) + machine_family = local.machine_vals[0] + machine_shared_core = length(local.machine_vals) <= 2 + machine_vcpus = try(parseint(local.machine_vals[2], 10), 1) + + smt_capable_family = !contains(["t2d", "t2a"], local.machine_family) + smt_capable_vcpu = local.machine_vcpus >= 2 + + smt_capable = local.smt_capable_family && local.smt_capable_vcpu && !local.machine_shared_core + set_threads_per_core = var.threads_per_core != null && (var.threads_per_core == 0 && local.smt_capable || try(var.threads_per_core >= 1, false)) + threads_per_core = var.threads_per_core == 2 ? 2 : 1 +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/variables.tf new file mode 100644 index 0000000000..5519b8cd40 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/variables.tf @@ -0,0 +1,452 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "instance_count" { + description = "Number of instances" + type = number + default = 1 +} + +variable "instance_image" { + description = "Instance Image" + type = map(string) + default = { + project = "cloud-hpc-image-public" + family = "hpc-rocky-linux-8" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "disk_size_gb" { + description = "Size of disk for instances." + type = number + default = 200 +} + +variable "disk_type" { + description = "Disk type for instances." + type = string + default = "pd-standard" +} + +variable "auto_delete_boot_disk" { + description = "Controls if boot disk should be auto-deleted when instance is deleted." + type = bool + default = true +} + +variable "local_ssd_count" { + description = "The number of local SSDs to attach to each VM. See https://cloud.google.com/compute/docs/disks/local-ssd." + type = number + default = 0 +} + +variable "local_ssd_interface" { + description = "Interface to be used with local SSDs. Can be either 'NVME' or 'SCSI'. No effect unless `local_ssd_count` is also set." + type = string + default = "NVME" +} + +variable "additional_persistent_disks" { + description = "Configurations of additional disks to be included on the partition nodes." + type = object({ + count = optional(number, 0) + type = optional(string, "pd-balanced") + size = optional(number, 200) + }) + default = {} +} + +variable "name_prefix" { + description = <<-EOT + An optional name for all VM and disk resources. + If not supplied, `deployment_name` will be used. + When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set, + then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". + EOT + type = string + default = null +} + +variable "add_deployment_name_before_prefix" { + description = <<-EOT + If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments. + See `name_prefix` for further details on resource naming behavior. + EOT + type = bool + default = false +} + +variable "disable_public_ips" { + description = "If set to true, instances will not have public IPs" + type = bool + default = false +} + +variable "machine_type" { + description = "Machine type to use for the instance creation" + type = string + default = "c2-standard-60" +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured." + type = list(object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "deployment_name" { + description = "Name of the deployment, will optionally be used name resources according to `name_prefix`" + type = string +} + +variable "labels" { + description = "Labels to add to the instances. Key-value pairs." + type = map(string) +} + +variable "service_account_email" { + description = "Service account e-mail address to use with the node pool" + type = string + default = null +} + +variable "service_account_scopes" { + description = "Scopes to to use with the node pool." + type = set(string) + default = ["https://www.googleapis.com/auth/cloud-platform"] +} + +# tflint-ignore: terraform_unused_declarations +variable "service_account" { + description = "DEPRECATED - Use `service_account_email` and `service_account_scopes` instead." + type = object({ + email = string, + scopes = set(string) + }) + default = null + validation { + condition = var.service_account == null + error_message = "The 'service_account' setting is deprecated, please use 'var.service_account_email' and 'var.service_account_scopes' instead." + } +} + +variable "network_self_link" { + description = "The self link of the network to attach the VM. Can use \"default\" for the default network." + type = string + default = null +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork to attach the VM." + type = string + default = null +} + +variable "network_interfaces" { + description = <<-EOT + A list of network interfaces. The options match that of the terraform + network_interface block of google_compute_instance. For descriptions of the + subfields or more information see the documentation: + https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface + + **_NOTE:_** If `network_interfaces` are set, `network_self_link` and + `subnetwork_self_link` will be ignored, even if they are provided through + the `use` field. `bandwidth_tier` and `disable_public_ips` also do not apply + to network interfaces defined in this variable. + + Subfields: + network (string, required if subnetwork is not supplied) + subnetwork (string, required if network is not supplied) + subnetwork_project (string, optional) + network_ip (string, optional) + nic_type (string, optional, choose from ["GVNIC", "VIRTIO_NET", "MRDMA", "IRDMA"]) + stack_type (string, optional, choose from ["IPV4_ONLY", "IPV4_IPV6"]) + queue_count (number, optional) + access_config (object, optional) + ipv6_access_config (object, optional) + alias_ip_range (list(object), optional) + EOT + type = list(object({ + network = string, + subnetwork = string, + subnetwork_project = string, + network_ip = string, + nic_type = string, + stack_type = string, + queue_count = number, + access_config = list(object({ + nat_ip = string, + public_ptr_domain_name = string, + network_tier = string + })), + ipv6_access_config = list(object({ + public_ptr_domain_name = string, + network_tier = string + })), + alias_ip_range = list(object({ + ip_cidr_range = string, + subnetwork_range_name = string + })) + })) + default = [] + validation { + condition = alltrue([ + for ni in var.network_interfaces : (ni.network == null) != (ni.subnetwork == null) + ]) + error_message = "All additional network interfaces must define exactly one of \"network\" or \"subnetwork\"." + } + validation { + condition = alltrue([ + for ni in var.network_interfaces : ni.nic_type == "GVNIC" || ni.nic_type == "VIRTIO_NET" || ni.nic_type == "MRDMA" || ni.nic_type == "IRDMA" || ni.nic_type == null + ]) + error_message = "In the variable network_interfaces, field \"nic_type\" must be \"GVNIC\", \"VIRTIO_NET\", \"MRDMA\", \"IRDMA\", or null." + } + validation { + condition = alltrue([ + for ni in var.network_interfaces : ni.stack_type == "IPV4_ONLY" || ni.stack_type == "IPV4_IPV6" || ni.stack_type == null + ]) + error_message = "In the variable network_interfaces, field \"stack_type\" must be either \"IPV4_ONLY\", \"IPV4_IPV6\" or null." + } +} + +variable "region" { + description = "The region to deploy to" + type = string +} + +variable "zone" { + description = "Compute Platform zone" + type = string +} + +variable "metadata" { + description = "Metadata, provided as a map" + type = map(string) + default = {} +} + +variable "startup_script" { + description = "Startup script used on the instance" + type = string + default = null +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = string, + count = number + })) + default = [] + nullable = false +} + +variable "automatic_restart" { + description = "Specifies if the instance should be restarted if it was terminated by Compute Engine (not a user)." + type = bool + default = null +} + +variable "on_host_maintenance" { + description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE`" + type = string + default = null + validation { + condition = var.on_host_maintenance == null ? true : contains(["MIGRATE", "TERMINATE"], var.on_host_maintenance) + error_message = "When set, the on_host_maintenance must be set to MIGRATE or TERMINATE." + } +} + +variable "bandwidth_tier" { + description = <= 0, false) && try(var.threads_per_core <= 2, false) + error_message = "Allowed values for threads_per_core are \"null\", \"0\", \"1\", \"2\"." + } + +} + +variable "enable_oslogin" { + description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." + type = string + default = "ENABLE" + validation { + condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) + error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." + } +} + +variable "allocate_ip" { + description = <<-EOT + If not null, allocate IPs with the given configuration. See details at + https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address + EOT + type = object({ + address_type = optional(string, "INTERNAL") + purpose = optional(string), + network_tier = optional(string), + ip_version = optional(string, "IPV4"), + }) + default = null +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} + +variable "reservation_name" { + description = <<-EOD + Name of the reservation to use for VM resources, should be in one of the following formats: + - projects/PROJECT_ID/reservations/RESERVATION_NAME + - RESERVATION_NAME + + Must be a "SPECIFIC_RESERVATION" + Set to empty string if using no reservation or automatically-consumed reservations + EOD + type = string + default = "" + nullable = false + + validation { + condition = length(regexall("^((projects/([a-z0-9-]+)/reservations/)?([a-z0-9-]+))?$", var.reservation_name)) > 0 + error_message = "Reservation name must be either empty or in the format '[projects/PROJECT_ID/reservations/]RESERVATION_NAME', [...] is an optional part." + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/versions.tf new file mode 100644 index 0000000000..4b2786e2af --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/versions.tf @@ -0,0 +1,41 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.73.0" + } + + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.13.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.57.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.57.0" + } + + required_version = ">= 1.3.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/README.md b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/README.md new file mode 100644 index 0000000000..0ee90e1d63 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/README.md @@ -0,0 +1,248 @@ +## Description + +This module creates a [filestore](https://cloud.google.com/filestore) +instance. Filestore is a high performance network file system that can be +mounted to one or more compute VMs. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). + +### Deletion protection + +We recommend considering enabling [Filestore deletion protection][fdp]. Deletion +protection will prevent unintentional deletion of an entire Filestore instance. +It does not prevent deletion of files within the Filestore instance when mounted +by a VM. It is not available on some [tiers](#filestore-tiers), including the +default BASIC\_HDD tier or BASIC\_SSD tier. Follow the documentation link for +up to date details. + +Usage can be enabled in a blueprint with, for example: + +```yaml + - id: homefs + source: modules/file-system/filestore + use: [network] + settings: + deletion_protection: + enabled: true + reason: Avoid data loss + filestore_tier: ZONAL + local_mount: /home + size_gb: 1024 +``` + +[fdp]: https://cloud.google.com/filestore/docs/deletion-protection + +### Filestore tiers + +At the time of writing, Filestore supports 5 [tiers of service][tiers] that are +specified in the Toolkit using the following names: + +- Basic HDD: "BASIC\_HDD" ([preferred][tierapi]) or "STANDARD" (deprecated) +- Basic SSD: "BASIC\_SSD" ([preferred][tierapi]) or "PREMIUM" (deprecated) +- Zonal: "ZONAL" +- Enterprise: "ENTERPRISE" +- Regional: "REGIONAL" + +[tierapi]: https://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier + +**Please review the minimum storage requirements for each tier**. The Terraform +module can only enforce the minimum value of the `size_gb` parameter for the +lowest tier of service. If you supply a value that is too low, Filestore +creation will fail when you run `terraform apply`. + +[tiers]: https://cloud.google.com/filestore/docs/service-tiers + +### Filestore protocols and mount options +After Filestore instance is created, you can mount this to the compute node +using different mount options. Toolkit uses [default mount options](https://linux.die.net/man/8/mount) +for all tier services. Filestore has recommended mount options for different +service tiers which may overall improve performance. These can be found here: +[recommended mount options.](https://cloud.google.com/filestore/docs/mounting-fileshares) +While creating filestore module, you can overwrite these mount options as +mentioned below. + +```yaml +- id: homefs + source: modules/file-system/filestore + use: [network1] + settings: + local_mount: /homefs + mount_options: defaults,hard,timeo=600,retrans=3,_netdev +``` + +Filestore supports NFS protocols `NFS_V3` (default) and `NFS_V4_1`. Protocol support depends on the selected tier: +- `NFS_V3`: Supported on all tiers (`BASIC_HDD`, `BASIC_SSD`, `HIGH_SCALE_SSD`, `ZONAL`, `ENTERPRISE`). +- `NFS_V4_1`: Supported only on `HIGH_SCALE_SSD`, `ZONAL`, `REGIONAL`, and `ENTERPRISE`. +This can be specified at creation time via the `protocol` variable. By default, `NFS_V3` is used for compatibility. +See the example below and [this page](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance#protocol-1) for more information. + +```yaml +- id: homefs + source: modules/file-system/filestore + use: [network1] + settings: + local_mount: /homefs + protocol: NFS_V4_1 + filestore_tier: ZONAL +``` + +### Filestore quota + +Your project must have unused quota for Cloud Filestore in the region you will +provision the storage. This can be found by browsing to the [Quota tab within IAM +& Admin](https://console.cloud.google.com/iam-admin/quotas) in the Cloud Console. +Please note that there are separate quota limits for HDD and SSD storage. + +All projects begin with 0 available quota for High Scale SSD tier. To use this +tier, [make a request and wait for it to be approved][hs-ssd-quota]. + +[hs-ssd-quota]: https://cloud.google.com/filestore/docs/high-scale + +### Example - Basic HDD + +The Filestore instance defined below will have the following attributes: + +- (default) `BASIC_HDD` tier +- (default) 1TiB capacity +- `homefs` module ID +- mount point at `/home` +- connected to the network defined in the `network1` module + +```yaml +- id: homefs + source: modules/file-system/filestore + use: [network1] + settings: + local_mount: /home +``` + +### Example - High Scale SSD + +The Filestore instance defined below will have the following attributes: + +- `HIGH_SCALE_SSD` tier +- 10TiB capacity +- `highscale` module ID +- mount point at `/projects` +- connected to the VPC network defined in the `network1` module + +```yaml +- id: highscale + source: modules/file-system/filestore + use: [network1] + settings: + filestore_tier: HIGH_SCALE_SSD + size_gb: 10240 + local_mount: /projects +``` + +## Mounting + +To mount the Filestore instance you must first ensure that the NFS client has +been installed and then call the proper `mount` command. + +Both of these steps are automatically handled with the use of the `use` command +in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in +the network storage doc for a complete list of supported modules. +See the [hpc-slurm](../../../examples/hpc-slurm.yaml) for +an example of using this module with Slurm. + +If mounting is not automatically handled as described above, the `filestore` +module outputs runners that can be used with the startup-script module to +install the client and mount the file system. See the following example: + +```yaml + - id: filestore + source: modules/file-system/filestore + use: [network1] + settings: {local_mount: /scratch} + + - id: mount-at-startup + source: modules/scripts/startup-script + settings: + runners: + - $(filestore.install_nfs_client_runner) + - $(filestore.mount_runner) + +``` + +[matrix]: ../../../docs/network_storage.md#compatibility-matrix + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [google](#requirement\_google) | >= 6.4 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.4 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_filestore_instance.filestore_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [connect\_mode](#input\_connect\_mode) | Used to select mode - supported values DIRECT\_PEERING and PRIVATE\_SERVICE\_ACCESS. | `string` | `"DIRECT_PEERING"` | no | +| [deletion\_protection](#input\_deletion\_protection) | Configure Filestore instance deletion protection | 
object({
    enabled = optional(bool, false)
    reason  = optional(string)
  })
| 
{
  "enabled": false
}
| no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the filestore instance if no name is specified. | `string` | n/a | yes | +| [description](#input\_description) | A description of the filestore instance. | `string` | `""` | no | +| [filestore\_share\_name](#input\_filestore\_share\_name) | Name of the file system share on the instance. | `string` | `"nfsshare"` | no | +| [filestore\_tier](#input\_filestore\_tier) | The service tier of the instance. | `string` | `"BASIC_HDD"` | no | +| [labels](#input\_labels) | Labels to add to the filestore instance. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_mount](#input\_local\_mount) | Mountpoint for this filestore instance. Note: If set to the same as the `filestore_share_name`, it will trigger a known Slurm bug ([troubleshooting](../../../docs/slurm-troubleshooting.md)). | `string` | `"/shared"` | no | +| [mount\_options](#input\_mount\_options) | NFS mount options to mount file system. | `string` | `"defaults,_netdev"` | no | +| [name](#input\_name) | The resource name of the instance. | `string` | `null` | no | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | +| [nfs\_export\_options](#input\_nfs\_export\_options) | Define NFS export options. | 
list(object({
    access_mode = optional(string)
    ip_ranges   = optional(list(string))
    squash_mode = optional(string)
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | ID of project in which Filestore instance will be created. | `string` | n/a | yes | +| [protocol](#input\_protocol) | NFS protocol version. Default is NFS\_V3. NFS\_V4\_1 is only supported with HIGH\_SCALE\_SSD, ZONAL, REGIONAL, and ENTERPRISE tiers. | `string` | `"NFS_V3"` | no | +| [region](#input\_region) | Location for Filestore instances at Enterprise tier. | `string` | n/a | yes | +| [reserved\_ip\_range](#input\_reserved\_ip\_range) | Reserved IP range for Filestore instance. Users are encouraged to set to null
for automatic selection. If supplied, it must be:

CIDR format when var.connect\_mode == "DIRECT\_PEERING"
Named IP Range when var.connect\_mode == "PRIVATE\_SERVICE\_ACCESS"

See Cloud documentation for more details:

https://cloud.google.com/filestore/docs/creating-instances#configure_a_reserved_ip_address_range | `string` | `null` | no | +| [size\_gb](#input\_size\_gb) | Storage size of the filestore instance in GB. | `number` | `1024` | no | +| [zone](#input\_zone) | Location for Filestore instances below Enterprise tier. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [capacity\_gb](#output\_capacity\_gb) | File share capacity in GiB. | +| [filestore\_id](#output\_filestore\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}` | +| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | +| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | +| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | +| [network\_storage](#output\_network\_storage) | Describes a filestore instance. | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/main.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/main.tf new file mode 100644 index 0000000000..ce035dbb2b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/main.tf @@ -0,0 +1,116 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "filestore", ghpc_role = "file-system" }) +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +locals { + is_high_capacity_tier = contains(["HIGH_SCALE_SSD", "ZONAL", "REGIONAL"], var.filestore_tier) && var.size_gb >= 10240 && var.size_gb <= 102400 + + timeouts = local.is_high_capacity_tier ? [1] : [] + server_ip = google_filestore_instance.filestore_instance.networks[0].ip_addresses[0] + remote_mount = format("/%s", google_filestore_instance.filestore_instance.file_shares[0].name) + fs_type = "nfs" + mount_options = var.mount_options + + install_nfs_client_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/install-nfs-client.sh" + "destination" = "install-nfs${replace(var.local_mount, "/", "_")}.sh" + } + mount_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/mount.sh" + "args" = "\"${local.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" + "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" + } + + # id format: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network#id + split_network_id = split("/", var.network_id) + network_name = local.split_network_id[4] + network_project = local.split_network_id[1] + shared_vpc = local.network_project != var.project_id +} + +resource "google_filestore_instance" "filestore_instance" { + project = var.project_id + + name = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" + description = var.description + location = contains(["ENTERPRISE", "REGIONAL"], var.filestore_tier) ? var.region : var.zone + tier = var.filestore_tier + protocol = var.protocol + + deletion_protection_enabled = var.deletion_protection.enabled + deletion_protection_reason = var.deletion_protection.reason + + file_shares { + capacity_gb = var.size_gb + name = var.filestore_share_name + dynamic "nfs_export_options" { + for_each = var.nfs_export_options + content { + access_mode = nfs_export_options.value.access_mode + ip_ranges = nfs_export_options.value.ip_ranges + squash_mode = nfs_export_options.value.squash_mode + } + } + } + + labels = local.labels + + networks { + network = local.shared_vpc ? var.network_id : local.network_name + connect_mode = var.connect_mode + modes = ["MODE_IPV4"] + reserved_ip_range = var.reserved_ip_range + } + + dynamic "timeouts" { + for_each = local.timeouts + content { + create = "1h" + update = "1h" + delete = "1h" + } + } + + lifecycle { + precondition { + condition = ( + var.reserved_ip_range == null || + var.connect_mode == "PRIVATE_SERVICE_ACCESS" || + var.connect_mode == "DIRECT_PEERING" && can(cidrhost(var.reserved_ip_range, 0)) && contains(["24", "29"], try(split("/", var.reserved_ip_range)[1], "")) + ) + error_message = <<-EOT + If connect_mode is set to DIRECT_PEERING and reserved_ip_range is + specified then it must be a CIDR IP range with suffix range size 29 for + BASIC_HDD or BASIC_SSD tiers. Otherwise the range size must be 24. + EOT + } + + precondition { + condition = !startswith(var.filestore_tier, "BASIC") || var.protocol != "NFS_V4_1" + error_message = "NFS_V4_1 is not supported on BASIC Filestore tiers." + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/metadata.yaml new file mode 100644 index 0000000000..5298336f09 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - file.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/outputs.tf new file mode 100644 index 0000000000..bd9126798c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/outputs.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "network_storage" { + description = "Describes a filestore instance." + value = { + server_ip = local.server_ip + remote_mount = local.remote_mount + local_mount = var.local_mount + fs_type = local.fs_type + mount_options = local.mount_options + client_install_runner = local.install_nfs_client_runner + mount_runner = local.mount_runner + } +} + +output "install_nfs_client" { + description = "Script for installing NFS client" + value = file("${path.module}/scripts/install-nfs-client.sh") +} + +output "install_nfs_client_runner" { + description = "Runner to install NFS client using the startup-script module" + value = local.install_nfs_client_runner +} + +output "mount_runner" { + description = <<-EOT + Runner to mount the file-system using an ansible playbook. The startup-script + module will automatically handle installation of ansible. + - id: example-startup-script + source: modules/scripts/startup-script + settings: + runners: + - $(your-fs-id.mount_runner) + ... + EOT + value = local.mount_runner +} + +output "filestore_id" { + description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}`" + value = google_filestore_instance.filestore_instance.id +} + +output "capacity_gb" { + description = "File share capacity in GiB." + value = google_filestore_instance.filestore_instance.file_shares[0].capacity_gb +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh new file mode 100644 index 0000000000..9f842c5d7c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! "$(which mount.nfs)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || + [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + major_version=$(rpm -E "%{rhel}") + enable_repo="" + if [ "${major_version}" -eq "7" ]; then + enable_repo="base,epel" + elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then + enable_repo="baseos" + else + echo "Unsupported version of centos/RHEL/Rocky" + return 1 + fi + yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils + elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get -y install nfs-common + else + echo 'Unsuported distribution' + return 1 + fi +fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/mount.sh b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/mount.sh new file mode 100644 index 0000000000..e2509fb4a1 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/mount.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SERVER_IP=$1 +REMOTE_MOUNT=$2 +LOCAL_MOUNT=$3 +FS_TYPE=$4 +MOUNT_OPTIONS=$5 + +[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" + +if [ "${FS_TYPE}" = "gcsfuse" ]; then + FS_SPEC="${REMOTE_MOUNT}" +else + FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" +fi + +SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" +EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" + +grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false +grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false +findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/mount.yaml b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/mount.yaml new file mode 100644 index 0000000000..f7fbe58d5e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/mount.yaml @@ -0,0 +1,39 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Mounts the file systems specified in the metadata network_storage key + hosts: localhost + become: true + vars: + meta_key: "network_storage" + url: "http://metadata.google.internal/computeMetadata/v1/instance/attributes" + tasks: + - name: Read metadata network_storage information + ansible.builtin.uri: + url: "{{ url }}/{{ meta_key }}" + method: GET + headers: + Metadata-Flavor: "Google" + register: storage + - name: Mount file systems + ansible.posix.mount: + src: "{{ item.server_ip }}:/{{ item.remote_mount }}" + path: "{{ item.local_mount }}" + opts: "{{ item.mount_options }}" + boot: true + fstype: "{{ item.fs_type }}" + state: "mounted" + loop: "{{ storage.json }}" diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/variables.tf new file mode 100644 index 0000000000..2d7e9258c0 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/variables.tf @@ -0,0 +1,189 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which Filestore instance will be created." + type = string +} + +variable "deployment_name" { + description = "Name of the HPC deployment, used as name of the filestore instance if no name is specified." + type = string +} + +variable "zone" { + description = "Location for Filestore instances below Enterprise tier." + type = string +} + +variable "region" { + description = "Location for Filestore instances at Enterprise tier." + type = string +} + +variable "network_id" { + description = <<-EOT + The ID of the GCE VPC network to which the instance is connected given in the format: + `projects//global/networks/`" + EOT + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "name" { + description = "The resource name of the instance." + type = string + default = null +} + +variable "filestore_share_name" { + description = "Name of the file system share on the instance." + type = string + default = "nfsshare" +} + +variable "local_mount" { + description = "Mountpoint for this filestore instance. Note: If set to the same as the `filestore_share_name`, it will trigger a known Slurm bug ([troubleshooting](../../../docs/slurm-troubleshooting.md))." + type = string + default = "/shared" +} + +variable "size_gb" { + description = "Storage size of the filestore instance in GB." + type = number + default = 1024 + validation { + condition = var.size_gb >= 1024 + error_message = "No Filestore tier supports less than 1024GiB.\nSee https://cloud.google.com/filestore/docs/service-tiers." + } +} + +variable "filestore_tier" { + description = "The service tier of the instance." + type = string + default = "BASIC_HDD" + validation { + condition = var.filestore_tier != "STANDARD" + error_message = "The preferred name for STANDARD tier is now BASIC_HDD\nhttps://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier." + } + validation { + condition = var.filestore_tier != "PREMIUM" + error_message = "The preferred name for PREMIUM tier is now BASIC_SSD\nhttps://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier." + } + validation { + condition = contains([ + "BASIC_HDD", + "BASIC_SSD", + "HIGH_SCALE_SSD", + "ZONAL", + "REGIONAL", + "ENTERPRISE" + ], var.filestore_tier) + # Avoid adding the legacy tier name in error_message, for e.g. 'HIGH_SCALE_SSD', 'ENTERPRISE'. + # As we want to steer the customer to new one's, but also support the legacy ones for older customers. + error_message = "Allowed values for filestore_tier are 'BASIC_HDD','BASIC_SSD','ZONAL','REGIONAL'.\nhttps://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance#tier\nhttps://cloud.google.com/filestore/docs/reference/rest/v1/Tier." + } +} + +variable "labels" { + description = "Labels to add to the filestore instance. Key-value pairs." + type = map(string) +} + +variable "connect_mode" { + description = "Used to select mode - supported values DIRECT_PEERING and PRIVATE_SERVICE_ACCESS." + type = string + default = "DIRECT_PEERING" + nullable = false + validation { + condition = contains(["DIRECT_PEERING", "PRIVATE_SERVICE_ACCESS"], var.connect_mode) + error_message = "Allowed values for connect_mode are \"DIRECT_PEERING\" or \"PRIVATE_SERVICE_ACCESS\"." + } +} + +variable "nfs_export_options" { + description = "Define NFS export options." + type = list(object({ + access_mode = optional(string) + ip_ranges = optional(list(string)) + squash_mode = optional(string) + })) + default = [] + nullable = false +} + +variable "reserved_ip_range" { + description = <<-EOT + Reserved IP range for Filestore instance. Users are encouraged to set to null + for automatic selection. If supplied, it must be: + + CIDR format when var.connect_mode == "DIRECT_PEERING" + Named IP Range when var.connect_mode == "PRIVATE_SERVICE_ACCESS" + + See Cloud documentation for more details: + + https://cloud.google.com/filestore/docs/creating-instances#configure_a_reserved_ip_address_range + EOT + type = string + default = null + nullable = true +} + +variable "mount_options" { + description = "NFS mount options to mount file system." + type = string + default = "defaults,_netdev" +} + +variable "deletion_protection" { + description = "Configure Filestore instance deletion protection" + type = object({ + enabled = optional(bool, false) + reason = optional(string) + }) + default = { + enabled = false + } + nullable = false + + validation { + condition = !can(coalesce(var.deletion_protection.reason)) || var.deletion_protection.enabled + error_message = "Cannot set Filestore var.deletion_protection.reason unless var.deletion_protection.enabled is true" + } +} + +variable "protocol" { + description = "NFS protocol version. Default is NFS_V3. NFS_V4_1 is only supported with HIGH_SCALE_SSD, ZONAL, REGIONAL, and ENTERPRISE tiers." + type = string + default = "NFS_V3" + validation { + condition = contains(["NFS_V3", "NFS_V4_1"], var.protocol) + error_message = "Allowed values for protocol are 'NFS_V3' or 'NFS_V4_1'." + } +} + +variable "description" { + description = "A description of the filestore instance." + type = string + default = "" + validation { + condition = length(var.description) <= 2048 + error_message = "Filestore description must be 2048 characters or fewer" + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/versions.tf new file mode 100644 index 0000000000..efde3ac158 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/versions.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.4" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.57.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.57.0" + } + + required_version = ">= 1.3.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/README.md b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/README.md new file mode 100644 index 0000000000..f4d94d8c3b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/README.md @@ -0,0 +1,166 @@ +## Description + +This module creates Kubernetes Persistent Volumes (PV) and Persistent Volume +Claims (PVC) that can be used by a [gke-job-template]. + +`gke-persistent-volume` works with Filestore and Google Cloud Storage. Each +`gke-persistent-volume` can only be used with a single file system so if multiple +shared file systems are used then multiple `gke-persistent-volume` modules are +needed in the blueprint. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +### Example + +The following example creates a Filestore and then uses the +`gke-persistent-volume` module to use the Filestore as shared storage in a +`gke-job-template`. + +```yaml + - id: gke_cluster + source: modules/scheduler/gke-cluster + use: [network1] + settings: + master_authorized_networks: + - display_name: deployment-machine + cidr_block: /32 + + - id: datafs + source: modules/file-system/filestore + use: [network1] + settings: + local_mount: /data + + - id: datafs-pv + source: modules/file-system/gke-persistent-volume + use: [datafs, gke_cluster] + + - id: job-template + source: modules/compute/gke-job-template + use: [datafs-pv, compute_pool] +``` + +The following example creates a GCS bucket and then uses the +`gke-persistent-volume` module to use the bucket as shared storage in a +`gke-job-template`. + +```yaml + - id: gke_cluster + source: modules/scheduler/gke-cluster + use: [network1] + settings: + master_authorized_networks: + - display_name: deployment-machine + cidr_block: /32 + + - id: data-bucket + source: community/modules/file-system/cloud-storage-bucket + settings: + local_mount: /data + + - id: datafs-pv + source: modules/file-system/gke-persistent-volume + use: [data-bucket, gke_cluster] + + - id: job-template + source: modules/compute/gke-job-template + use: [datafs-pv, compute_pool, gke_cluster] +``` + +See example +[storage-gke.yaml](../../../../examples/README.md#storage-gkeyaml--) blueprint +for a complete example. + +### Authorized Network + +Since the `gke-persistent-volume` module is making calls to the Kubernetes API +to create Kubernetes entities, the machine performing the deployment must be +authorized to connect to the Kubernetes API. You can add the +`master_authorized_networks` settings block, as shown in the example above, with +the IP address of the machine performing the deployment. This will ensure that +the deploying machine can connect to the cluster. + +### Connecting Via Use + +The diagram below shows the valid `use` relationships for the GKE Cluster Toolkit +modules. For example the `gke-persistent-volume` module can `use` a +`gke-cluster` module and a `filestore` module, as shown in the example above. + +```mermaid +graph TD; + vpc-->|OneToMany|gke-cluster; + gke-cluster-->|OneToMany|gke-node-pool; + gke-node-pool-->|ManyToMany|gke-job-template; + gke-cluster-->|OneToMany|gke-persistent-volume; + gke-persistent-volume-->|ManyToMany|gke-job-template; + vpc-->|OneToMany|filestore; + filestore-->|OneToOne|gke-persistent-volume; +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | >= 4.42 | +| [kubectl](#requirement\_kubectl) | >= 1.7.0 | +| [local](#requirement\_local) | >= 2.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.42 | +| [kubectl](#provider\_kubectl) | >= 1.7.0 | +| [local](#provider\_local) | >= 2.0.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [kubectl_manifest.pv](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.pvc](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [local_file.debug_file](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | +| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [capacity\_gb](#input\_capacity\_gb) | The storage capacity with which to create the persistent volume. | `number` | n/a | yes | +| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}` | `string` | n/a | yes | +| [filestore\_id](#input\_filestore\_id) | An identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `string` | `null` | no | +| [gcs\_bucket\_name](#input\_gcs\_bucket\_name) | The gcs bucket to be used with the persistent volume. | `string` | `null` | no | +| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | +| [network\_storage](#input\_network\_storage) | Network attached storage mount to be configured. | 
object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  })
| n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [persistent\_volume\_claims](#output\_persistent\_volume\_claims) | An object that describes a k8s PVC created by this module. | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/main.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/main.tf new file mode 100644 index 0000000000..3691523148 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/main.tf @@ -0,0 +1,124 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "gke-persistent-volume", ghpc_role = "file-system" }) +} + +locals { + is_gcs = (var.gcs_bucket_name != null) + + filestore_id = ( + !local.is_gcs ? # If not using gcs + var.filestore_id : # Then filestore_id must be provided + "projects/empty/locations/empty/instances/empty" # Otherwise use something arbitrary as it will not be used + ) + location = split("/", local.filestore_id)[3] + filestore_name = split("/", local.filestore_id)[5] + filestore_share_name = trimprefix(var.network_storage.remote_mount, "/") + base_name = local.is_gcs ? var.gcs_bucket_name : local.filestore_name + + pv_name = "${local.base_name}-pv" + pvc_name = "${local.base_name}-pvc" + + list_mount_options = split(",", var.network_storage.mount_options) + + filestore_pv_contents = templatefile( + "${path.module}/templates/filestore-pv.yaml.tftpl", + { + pv_name = local.pv_name + capacity = "${var.capacity_gb}Gi" + location = local.location + filestore_name = local.filestore_name + share_name = local.filestore_share_name + ip_address = var.network_storage.server_ip + labels = local.labels + } + ) + + filestore_pvc_contents = templatefile( + "${path.module}/templates/filestore-pvc.yaml.tftpl", + { + pv_name = local.pv_name + capacity = "${var.capacity_gb}Gi" + pvc_name = local.pvc_name + labels = local.labels + } + ) + + gcs_pv_contents = templatefile( + "${path.module}/templates/gcs-pv.yaml.tftpl", + { + pv_name = local.pv_name + capacity = "${var.capacity_gb}Gi" + labels = local.labels + mount_options = local.is_gcs ? local.list_mount_options : null + bucket_name = local.is_gcs ? var.gcs_bucket_name : "" + } + ) + + gcs_pvc_contents = templatefile( + "${path.module}/templates/gcs-pvc.yaml.tftpl", + { + pv_name = local.pv_name + pvc_name = local.pvc_name + labels = local.labels + capacity = "${var.capacity_gb}Gi" + } + ) + + cluster_name = split("/", var.cluster_id)[5] + cluster_location = split("/", var.cluster_id)[3] +} + +resource "local_file" "debug_file" { + content = <<-EOF + ${local.filestore_pv_contents} + ${local.filestore_pvc_contents} + EOF + filename = "${path.root}/pv-pvc-debug-file-${local.filestore_name}.yaml" +} + +data "google_container_cluster" "gke_cluster" { + name = local.cluster_name + location = local.cluster_location +} + +data "google_client_config" "default" {} + +provider "kubectl" { + host = "https://${data.google_container_cluster.gke_cluster.endpoint}" + cluster_ca_certificate = base64decode(data.google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate) + token = data.google_client_config.default.access_token + load_config_file = false +} + +resource "kubectl_manifest" "pv" { + yaml_body = local.is_gcs ? local.gcs_pv_contents : local.filestore_pv_contents + + lifecycle { + precondition { + condition = (var.gcs_bucket_name != null) != (var.filestore_id != null) + error_message = "Either gcs_bucket_name or filestore_id must be set." + } + } +} + +resource "kubectl_manifest" "pvc" { + yaml_body = local.is_gcs ? local.gcs_pvc_contents : local.filestore_pvc_contents + depends_on = [kubectl_manifest.pv] +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf new file mode 100644 index 0000000000..3c7d1b7050 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "persistent_volume_claims" { + description = "An object that describes a k8s PVC created by this module." + value = { + name = local.pvc_name + mount_path = var.network_storage.local_mount + mount_options = var.network_storage.mount_options + is_gcs = local.is_gcs + } + depends_on = [kubectl_manifest.pvc] +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl new file mode 100644 index 0000000000..cfda33978c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl @@ -0,0 +1,23 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: ${pv_name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + storageClassName: "" + capacity: + storage: ${capacity} + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + volumeMode: Filesystem + csi: + driver: filestore.csi.storage.gke.io + volumeHandle: "modeInstance/${location}/${filestore_name}/${share_name}" + volumeAttributes: + ip: ${ip_address} + volume: ${share_name} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl new file mode 100644 index 0000000000..5bcd735807 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl @@ -0,0 +1,17 @@ +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: ${pvc_name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ReadWriteMany + storageClassName: "" + volumeName: ${pv_name} + resources: + requests: + storage: ${capacity} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl new file mode 100644 index 0000000000..aa0e570a8b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl @@ -0,0 +1,24 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: ${pv_name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + storageClassName: "" + capacity: + storage: ${capacity} + accessModes: + - ReadWriteMany + %{~ if mount_options != null ~} + mountOptions: + %{~ for key in mount_options ~} + - ${key} + %{~ endfor ~} + %{~ endif ~} + csi: + driver: gcsfuse.csi.storage.gke.io + volumeHandle: ${bucket_name} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl new file mode 100644 index 0000000000..9af95b29d7 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${pvc_name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ReadWriteMany + storageClassName: "" + volumeName: ${pv_name} + resources: + requests: + storage: ${capacity} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf new file mode 100644 index 0000000000..a72fa3857f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "cluster_id" { + description = "An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}`" + type = string +} + +variable "network_storage" { + description = "Network attached storage mount to be configured." + type = object({ + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + }) +} + +variable "filestore_id" { + description = "An identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." + type = string + default = null + validation { + condition = ( + var.filestore_id == null || + try(length(split("/", var.filestore_id)), 0) == 6 + ) + error_message = "filestore_id must be in the format of 'projects/{{project}}/locations/{{location}}/instances/{{name}}'." + } +} + +variable "gcs_bucket_name" { + description = "The gcs bucket to be used with the persistent volume." + type = string + default = null +} + +variable "capacity_gb" { + description = "The storage capacity with which to create the persistent volume." + type = number +} + +variable "labels" { + description = "GCE resource labels to be applied to resources. Key-value pairs." + type = map(string) +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf new file mode 100644 index 0000000000..1c6b08e279 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf @@ -0,0 +1,34 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.42" + } + kubectl = { + source = "gavinbunney/kubectl" + version = ">= 1.7.0" + } + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:gke-persistent-volume/v1.57.0" + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/README.md b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/README.md new file mode 100644 index 0000000000..73396f6a03 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/README.md @@ -0,0 +1,133 @@ +## Description + +This module creates Kubernetes Storage Class (SC) that can be used by a Persistent Volume Claim (PVC) +to dynamically provision GCP storage resources like Parallelstore. + +### Example + +The following example uses the `gke-storage` module to creates a Parallelstore Storage Class and Persistent Volume Claim, +then use them in a `gke-job-template` to dynamically provision the resource. + +```yaml + - id: gke_cluster + source: modules/scheduler/gke-cluster + use: [network] + settings: + enable_parallelstore_csi: true + + # Private Service Access (PSA) requires the compute.networkAdmin role which is + # included in the Owner role, but not Editor. + # PSA is required for all Parallelstore functionality. + # https://cloud.google.com/vpc/docs/configure-private-services-access#permissions + - id: private_service_access + source: community/modules/network/private-service-access + use: [network] + settings: + prefix_length: 24 + + - id: gke_storage + source: modules/file-system/gke-storage + use: [ gke_cluster, private_service_access ] + settings: + storage_type: Parallelstore + access_mode: ReadWriteMany + sc_volume_binding_mode: Immediate + sc_reclaim_policy: Delete + sc_topology_zones: [$(vars.zone)] + pvc_count: 2 + capacity_gb: 12000 + + - id: job_template + source: modules/compute/gke-job-template + use: [gke_storage, compute_pool] +``` + +See example +[gke-managed-parallelstore.yaml](../../../examples/README.md#gke-managed-parallelstoreyaml--) blueprint +for a complete example. + +### Authorized Network + +Since the `gke-storage` module is making calls to the Kubernetes API +to create Kubernetes entities, the machine performing the deployment must be +authorized to connect to the Kubernetes API. You can add the +`master_authorized_networks` settings block, as shown in the example above, with +the IP address of the machine performing the deployment. This will ensure that +the deploying machine can connect to the cluster. + +### Connecting Via Use + +The diagram below shows the valid `use` relationships for the GKE Cluster Toolkit +modules. For example the `gke-storage` module can `use` a +`gke-cluster` module and a `private_service_access` module, as shown in the example above. + +```mermaid +graph TD; + vpc-->|OneToMany|gke-cluster; + gke-cluster-->|OneToMany|gke-node-pool; + gke-node-pool-->|ManyToMany|gke-job-template; + gke-cluster-->|OneToMany|gke-storage; + gke-storage-->|ManyToMany|gke-job-template; +``` + +## License + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_mode](#input\_access\_mode) | The access mode that the volume can be mounted to the host/pod. More details in [Access Modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes)
Valid access modes:
- ReadWriteOnce
- ReadOnlyMany
- ReadWriteMany
- ReadWriteOncePod | `string` | n/a | yes | +| [capacity\_gb](#input\_capacity\_gb) | The storage capacity with which to create the persistent volume. | `number` | n/a | yes | +| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}` | `string` | n/a | yes | +| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | +| [mount\_options](#input\_mount\_options) | Controls the mountOptions for dynamically provisioned PersistentVolumes of this storage class. | `string` | `null` | no | +| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). | `string` | `null` | no | +| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | +| [pv\_mount\_path](#input\_pv\_mount\_path) | Path within the container at which the volume should be mounted. Must not contain ':'. | `string` | `"/data"` | no | +| [pvc\_count](#input\_pvc\_count) | How many PersistentVolumeClaims that will be created | `number` | `1` | no | +| [sc\_reclaim\_policy](#input\_sc\_reclaim\_policy) | Indicate whether to keep the dynamically provisioned PersistentVolumes of this storage class after the bound PersistentVolumeClaim is deleted.
[More details about reclaiming](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming)
Supported value:
- Retain
- Delete | `string` | n/a | yes | +| [sc\_topology\_zones](#input\_sc\_topology\_zones) | Zone location that allow the volumes to be dynamically provisioned. | `list(string)` | `null` | no | +| [sc\_volume\_binding\_mode](#input\_sc\_volume\_binding\_mode) | Indicates when volume binding and dynamic provisioning should occur and how PersistentVolumeClaims should be provisioned and bound.
Supported value:
- Immediate
- WaitForFirstConsumer | `string` | `"WaitForFirstConsumer"` | no | +| [storage\_type](#input\_storage\_type) | The type of [GKE supported storage options](https://cloud.google.com/kubernetes-engine/docs/concepts/storage-overview)
to used. This module currently support dynamic provisioning for the below storage options
- Parallelstore
- Hyperdisk-balanced
- Hyperdisk-throughput
- Hyperdisk-extreme | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [persistent\_volume\_claims](#output\_persistent\_volume\_claims) | An object that describes a k8s PVC created by this module. | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/main.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/main.tf new file mode 100644 index 0000000000..18f85fa779 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/main.tf @@ -0,0 +1,78 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "gke-storage", ghpc_role = "file-system" }) +} + +locals { + storage_type = lower(var.storage_type) + storage_class_name = "${local.storage_type}-sc" + pvc_name_prefix = "${local.storage_type}-pvc" +} + +check "private_vpc_connection_peering" { + assert { + condition = lower(var.storage_type) != "parallelstore" ? true : var.private_vpc_connection_peering != null + error_message = <<-EOT + Parallelstore must be run within the same VPC as the GKE cluster and have private services access enabled. + If using new VPC, please use community/modules/network/private-service-access to create private-service-access. + If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). + EOT + } +} + +module "kubectl_apply" { + source = "../../management/kubectl-apply" + + cluster_id = var.cluster_id + project_id = var.project_id + + # count = var.pvc_count + apply_manifests = flatten( + [ + # create StorageClass in the cluster + { + content = templatefile( + "${path.module}/storage-class/${local.storage_class_name}.yaml.tftpl", + { + name = local.storage_class_name + labels = local.labels + volume_binding_mode = var.sc_volume_binding_mode + reclaim_policy = var.sc_reclaim_policy + topology_zones = var.sc_topology_zones + }) + }, + # create PersistentVolumeClaim in the cluster + flatten([ + for idx in range(var.pvc_count) : [ + { + content = templatefile( + "${path.module}/persistent-volume-claim/${(local.pvc_name_prefix)}.yaml.tftpl", + { + pvc_name = "${local.pvc_name_prefix}-${idx}" + labels = local.labels + capacity = "${var.capacity_gb}Gi" + access_mode = var.access_mode + storage_class_name = local.storage_class_name + } + ) + } + ] + ]) + ]) +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/metadata.yaml new file mode 100644 index 0000000000..8722823274 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/outputs.tf new file mode 100644 index 0000000000..b789674814 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/outputs.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "persistent_volume_claims" { + description = "An object that describes a k8s PVC created by this module." + value = flatten([ + for idx in range(var.pvc_count) : [{ + name = "${local.pvc_name_prefix}-${idx}" + mount_path = "${var.pv_mount_path}/${local.pvc_name_prefix}-${idx}" + mount_options = var.mount_options + is_gcs = false + }] + ]) +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl new file mode 100644 index 0000000000..32781be2fb --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${pvc_name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ${access_mode} + resources: + requests: + storage: ${capacity} + storageClassName: ${storage_class_name} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl new file mode 100644 index 0000000000..32781be2fb --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${pvc_name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ${access_mode} + resources: + requests: + storage: ${capacity} + storageClassName: ${storage_class_name} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl new file mode 100644 index 0000000000..32781be2fb --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${pvc_name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ${access_mode} + resources: + requests: + storage: ${capacity} + storageClassName: ${storage_class_name} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl new file mode 100644 index 0000000000..32781be2fb --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${pvc_name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +spec: + accessModes: + - ${access_mode} + resources: + requests: + storage: ${capacity} + storageClassName: ${storage_class_name} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl new file mode 100644 index 0000000000..46e1f023d3 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl @@ -0,0 +1,25 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +provisioner: pd.csi.storage.gke.io +allowVolumeExpansion: true +parameters: + type: hyperdisk-balanced + provisioned-throughput-on-create: "250Mi" + provisioned-iops-on-create: "7000" +volumeBindingMode: ${volume_binding_mode} +reclaimPolicy: ${reclaim_policy} + %{~ if topology_zones != null ~} +allowedTopologies: +- matchLabelExpressions: + - key: topology.gke.io/zone + values: + %{~ for z in topology_zones ~} + - ${z} + %{~ endfor ~} + %{~ endif ~} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl new file mode 100644 index 0000000000..445020d001 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl @@ -0,0 +1,24 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} +provisioner: pd.csi.storage.gke.io +allowVolumeExpansion: true +parameters: + %{~ endfor ~} + type: hyperdisk-extreme + provisioned-iops-on-create: "50000" +volumeBindingMode: ${volume_binding_mode} +reclaimPolicy: ${reclaim_policy} + %{~ if topology_zones != null ~} +allowedTopologies: +- matchLabelExpressions: + - key: topology.gke.io/zone + values: + %{~ for z in topology_zones ~} + - ${z} + %{~ endfor ~} + %{~ endif ~} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl new file mode 100644 index 0000000000..ec404aec45 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl @@ -0,0 +1,24 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +provisioner: pd.csi.storage.gke.io +allowVolumeExpansion: true +parameters: + type: hyperdisk-throughput + provisioned-throughput-on-create: "250Mi" +volumeBindingMode: ${volume_binding_mode} +reclaimPolicy: ${reclaim_policy} + %{~ if topology_zones != null ~} +allowedTopologies: +- matchLabelExpressions: + - key: topology.gke.io/zone + values: + %{~ for z in topology_zones ~} + - ${z} + %{~ endfor ~} + %{~ endif ~} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl new file mode 100644 index 0000000000..e6b8ea8d3e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl @@ -0,0 +1,21 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${name} + labels: + %{~ for key, val in labels ~} + ${key}: ${val} + %{~ endfor ~} +provisioner: parallelstore.csi.storage.gke.io +parameters: +volumeBindingMode: ${volume_binding_mode} +reclaimPolicy: ${reclaim_policy} + %{~ if topology_zones != null ~} +allowedTopologies: +- matchLabelExpressions: + - key: topology.gke.io/zone + values: + %{~ for z in topology_zones ~} + - ${z} + %{~ endfor ~} + %{~ endif ~} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/variables.tf new file mode 100644 index 0000000000..9f6224bbac --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/variables.tf @@ -0,0 +1,138 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "The project ID to host the cluster in." + type = string +} + +variable "cluster_id" { + description = "An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}`" + type = string +} + +variable "labels" { + description = "GCE resource labels to be applied to resources. Key-value pairs." + type = map(string) +} + +variable "storage_type" { + description = <<-EOT + The type of [GKE supported storage options](https://cloud.google.com/kubernetes-engine/docs/concepts/storage-overview) + to used. This module currently support dynamic provisioning for the below storage options + - Parallelstore + - Hyperdisk-balanced + - Hyperdisk-throughput + - Hyperdisk-extreme + EOT + type = string + nullable = false + validation { + condition = var.storage_type == null ? false : contains(["parallelstore", "hyperdisk-balanced", "hyperdisk-throughput", "hyperdisk-extreme"], lower(var.storage_type)) + error_message = "Allowed string values for var.storage_type are \"Parallelstore\", \"Hyperdisk-balanced\", \"Hyperdisk-throughput\", \"Hyperdisk-extreme\"." + } +} + +variable "access_mode" { + description = <<-EOT + The access mode that the volume can be mounted to the host/pod. More details in [Access Modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) + Valid access modes: + - ReadWriteOnce + - ReadOnlyMany + - ReadWriteMany + - ReadWriteOncePod + EOT + type = string + nullable = false + validation { + condition = var.access_mode == null ? false : contains(["readwriteonce", "readonlymany", "readwritemany", "readwriteoncepod"], lower(var.access_mode)) + error_message = "Allowed string values for var.access_mode are \"ReadWriteOnce\", \"ReadOnlyMany\", \"ReadWriteMany\", \"ReadWriteOncePod\"." + } +} + +variable "sc_volume_binding_mode" { + description = <<-EOT + Indicates when volume binding and dynamic provisioning should occur and how PersistentVolumeClaims should be provisioned and bound. + Supported value: + - Immediate + - WaitForFirstConsumer + EOT + type = string + default = "WaitForFirstConsumer" + validation { + condition = var.sc_volume_binding_mode == null ? true : contains(["immediate", "waitforfirstconsumer"], lower(var.sc_volume_binding_mode)) + error_message = "Allowed string values for var.sc_volume_binding_mode are \"Immediate\", \"WaitForFirstConsumer\"." + } +} + +variable "sc_reclaim_policy" { + description = <<-EOT + Indicate whether to keep the dynamically provisioned PersistentVolumes of this storage class after the bound PersistentVolumeClaim is deleted. + [More details about reclaiming](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming) + Supported value: + - Retain + - Delete + EOT + type = string + nullable = false + validation { + condition = var.sc_reclaim_policy == null ? true : contains(["retain", "delete"], lower(var.sc_reclaim_policy)) + error_message = "Allowed string values for var.sc_reclaim_policy are \"Retain\", \"Delete\"." + } +} + +variable "sc_topology_zones" { + description = "Zone location that allow the volumes to be dynamically provisioned." + type = list(string) + default = null +} + +variable "pvc_count" { + description = "How many PersistentVolumeClaims that will be created" + type = number + default = 1 +} + +variable "pv_mount_path" { + description = "Path within the container at which the volume should be mounted. Must not contain ':'." + type = string + default = "/data" + validation { + condition = var.pv_mount_path == null ? true : !strcontains(var.pv_mount_path, ":") + error_message = "pv_mount_path must not contain ':', please correct it and retry" + } +} + +variable "mount_options" { + description = "Controls the mountOptions for dynamically provisioned PersistentVolumes of this storage class." + type = string + default = null +} + +variable "capacity_gb" { + description = "The storage capacity with which to create the persistent volume." + type = number +} + +variable "private_vpc_connection_peering" { + description = <<-EOT + The name of the VPC Network peering connection. + If using new VPC, please use community/modules/network/private-service-access to create private-service-access and + If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). + EOT + type = string + default = null +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/versions.tf new file mode 100644 index 0000000000..ab0ea408ce --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/versions.tf @@ -0,0 +1,21 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.5" + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:gke-storage/v1.57.0" + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/README.md b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/README.md new file mode 100644 index 0000000000..e9c7f66268 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/README.md @@ -0,0 +1,289 @@ +## Description + +This module creates a [Managed Lustre](https://cloud.google.com/managed-lustre) +instance. Managed Lustre is a high performance network file system that can be +mounted to one or more VMs. + +For more information on this and other network storage options in the Cluster +Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). + +### Supported Operating Systems + +A Managed Lustre instance can be used with Slurm cluster or compute +VM running Ubuntu 20.04, 22.04 or Rocky Linux 8 (including the HPC flavor). + +### Managed Lustre Access + +Managed Lustre must be enabled for your project by Google staff. Please contact +your sales representative for further steps. + +### Example - New VPC + +For Managed Lustre instance, the snippet below creates new VPC and configures +private-service-access for this newly created network. Both items are required +to be passed to the Lustre module to ensure that they're built in order and +that the correct subnetwork has private service access. + +```yaml + - id: network + source: modules/network/vpc + + - id: private_service_access + source: community/modules/network/private-service-access + use: [network] + settings: + prefix_length: 24 + + - id: lustre + source: modules/file-system/managed-lustre + use: [network, private_service_access] +``` + +### Example - Slurm + +When using Slurm you must take into consideration whether or not you are using +an official image from the `schedmd-slurm-public` project or building your own. +The Lustre client modules are pre-installed in the official images. With the +official images, Lustre can be used as follows: + +```yaml +- id: managed_lustre + source: modules/file-system/managed-lustre + use: [network, private_service_access] + settings: + name: lustre-instance + local_mount: /lustre + remote_mount: lustrefs + size_gib: 18000 + +# Other modules: nodesets, partitions, login, etc. + +- id: slurm_controller + source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller + use: + - network + - lustre_partition + - managed_lustre + - slurm_login + settings: + machine_type: n2-standard-4 + enable_controller_public_ips: true +``` + +For custom images you must install the modules during the image build as the +Slurm cluster will not run the installation script like it does for the +standard VMs. + +Assuming you have a startup script for the Slurm image building, you can add +this Ansible playbook to correctly install the Lustre drivers into the image +(for Slurm-GCP versions greater than 6.10.0): + +```yaml +- type: data + destination: /var/tmp/slurm_vars.json + content: | + { + "reboot": false, + "install_cuda": false, + "install_gcsfuse": true, + "install_lustre": false, + "install_managed_lustre": true, + "install_nvidia_repo": true, + "install_ompi": true, + "allow_kernel_upgrades": false, + "monitoring_agent": "cloud-ops", + } +``` + +The `install_managed_lustre: true` line specifies that slurm-gcp should install +the correct modules within the slurm image. This runner should be placed +ahead of the script that calls the ansible build of the slurm-gcp image. + +### Example - Existing VPC + +If you want to use existing network with private-service-access configured, you need +to manually provide `private_vpc_connection_peering` to the Managed Lustre module. +You can get this details from the Google Cloud Console UI in `VPC network peering` +section. Below is the example of using existing network and creating Managed Lustre. +If existing network is not configured with private-service-access, you can follow +[Configure private service access](https://cloud.google.com/vpc/docs/configure-private-services-access) +to set it up. + +```yaml + - id: network + source: modules/network/pre-existing-vpc + settings: + network_name: // Add network name + subnetwork_name: // Add subnetwork name + + - id: lustre + source: modules/file-system/managed-lustre + use: [network] + settings: + private_vpc_connection_peering: # will look like "servicenetworking.googleapis.com" +``` + +### Example - GKE compatibility + +By default the Managed Lustre instance that is deployed is not compatible with +GKE. To enable the compatibility use the `gke_support_enabled: true` option. +This creates a file `/etc/modprobe/lnet.conf` that changes the listening port +to 6988. + +```yaml + - id: managed-lustre + source: modules/file-system/managed-lustre + use: [network, private_service_access] + settings: + name: lustre-instance + local_mount: /lustre + remote_mount: lustrefs + size_gib: 18000 + gke_support_enabled: true +``` + +> [!WARNING] +> +> 1. VMs cannot connect to both GKE compatible and GKE incompatible lustre +> instances at the same time as they connect to different ports. Lustre can +> only listen to one port at a time. +> +> 2. Setting `gke_support_enabled: true` will not affect Slurm nodes, GKE +> compatibility must be built into the Slurm image. + +### Example - Importing data from GSC Bucket + +One option with the Managed Lustre instance is to import data from a GSC bucket +upon the lustre instance creation. To do this, use the `import_gcs_bucket_uri` +variable to dictate the bucket to pull data from. The data will be imported +under the directory specified by `local_mount` (`/shared` if unspecified). + +> [!NOTE] +> +> 1. This is a one way operation. Once the data has been copied to the lustre +> instance it will not be updated with any changes made to the GCS bucket. +> +> 2. Once the lustre instance has been created in Terraform, the copy process +> will proceed in the background. Data may not be appear in the mounted +> directory for a period of time after the deployment has completed (see below). + +```yaml +- id: managed_lustre + source: modules/file-system/managed-lustre + use: [network, private_service_access] + settings: + name: lustre-instance + local_mount: /lustre + remote_mount: lustrefs + size_gib: 18000 + import_gcs_bucket_uri: gs:// +``` + +> [!WARNING] +> Please follow [this guide](https://cloud.google.com/managed-lustre/docs/transfer-data#required_permissions) +> to set up the correct IAM permissions for importing data from GCS to lustre. +> Without this, the copy process may fail silently leaving an empty lustre +> instance. + +If an import is requested, gcluster will output a json response similar to: + +```json +{ + "name": "projects//locations//operations/", + "metadata": { + "@type": "type.googleapis.com/google.cloud.lustre.v1.ImportDataMetadata", + "createTime": "", + "target": "projects//locations//instances/", + "requestedCancellation": false, + "apiVersion": "v1" + }, + "done": false +} +``` + +You can retrieve more information about the transfer using the following +command, substituting with values from the json response above: + +```bash +gcloud lustre operations describe --location --project +``` + +This will provide information on if the transfer is complete or if any errors +have occurred. See more at +[Get operation](https://cloud.google.com/managed-lustre/docs/transfer-data#get_operation). + +## License + + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [google](#requirement\_google) | >= 6.27.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.27.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_lustre_instance.lustre_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/lustre_instance) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [google_compute_network_peering.private_peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network_peering) | data source | +| [google_storage_bucket.lustre_import_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the Lustre instance if no name is specified. | `string` | n/a | yes | +| [description](#input\_description) | Description of the created Lustre instance. | `string` | `"Lustre Instance"` | no | +| [gke\_support\_enabled](#input\_gke\_support\_enabled) | Set to true to create Managed Lustre instance with GKE compatibility.
Note: This does not work with Slurm, the Slurm image must be built with
the correct compatibility. | `bool` | `false` | no | +| [import\_gcs\_bucket\_uri](#input\_import\_gcs\_bucket\_uri) | The name of the GCS bucket to import data from to managed lustre. Data will
be imported to the local\_mount directory. Changing this value will not
trigger a redeployment, to prevent data deletion. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to the Managed Lustre instance. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_mount](#input\_local\_mount) | Local mount point for the Managed Lustre instance. | `string` | `"/shared"` | no | +| [mount\_options](#input\_mount\_options) | Mounting options for the file system. | `string` | `"defaults,_netdev"` | no | +| [name](#input\_name) | Name of the Lustre instance | `string` | n/a | yes | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | +| [network\_self\_link](#input\_network\_self\_link) | Network self-link this instance will be on, required for checking private service access | `string` | n/a | yes | +| [per\_unit\_storage\_throughput](#input\_per\_unit\_storage\_throughput) | Throughput of the instance in MB/s/TiB. Valid values are 125, 250, 500, 1000. | `number` | `1000` | no | +| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually." | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | ID of project in which Lustre instance will be created. | `string` | n/a | yes | +| [remote\_mount](#input\_remote\_mount) | Remote mount point of the Managed Lustre instance | `string` | n/a | yes | +| [size\_gib](#input\_size\_gib) | Storage size of the Managed Lustre instance in GB. See https://cloud.google.com/managed-lustre/docs/create-instance for limitations | `number` | `18000` | no | +| [zone](#input\_zone) | Location for the Lustre instance. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [capacity\_gib](#output\_capacity\_gib) | File share capacity in GiB. | +| [install\_managed\_lustre\_client](#output\_install\_managed\_lustre\_client) | Script for installing Managed Lustre client | +| [lustre\_id](#output\_lustre\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}` | +| [network\_storage](#output\_network\_storage) | Describes a Managed Lustre instance. | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/main.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/main.tf new file mode 100644 index 0000000000..a969c53673 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/main.tf @@ -0,0 +1,104 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "managed-lustre", ghpc_role = "file-system" }) +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +data "google_compute_network_peering" "private_peering" { + name = var.private_vpc_connection_peering + network = var.network_self_link +} + +locals { + server_ip = split(":", google_lustre_instance.lustre_instance.mount_point)[0] + remote_mount = split(":", google_lustre_instance.lustre_instance.mount_point)[1] + fs_type = "lustre" + mount_options = var.mount_options + instance_id = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" + destination_path = "/" + + install_managed_lustre_client_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/install-managed-lustre-client.sh" + "destination" = "install-managed-lustre-client${replace(var.local_mount, "/", "_")}.sh" + "args" = var.gke_support_enabled ? "1" : "0" + } + mount_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/mount.sh" + "args" = "\"${local.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" + "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" + } + + bucket_count = try(length(data.google_storage_bucket.lustre_import_bucket), 0) +} + +data "google_storage_bucket" "lustre_import_bucket" { + count = try(length(var.import_gcs_bucket_uri) > 0, false) ? 1 : 0 + + name = split("//", var.import_gcs_bucket_uri)[1] +} + +resource "google_lustre_instance" "lustre_instance" { + project = var.project_id + + description = var.description + instance_id = local.instance_id + location = var.zone + + filesystem = var.remote_mount + capacity_gib = var.size_gib + per_unit_storage_throughput = var.per_unit_storage_throughput + + labels = local.labels + network = var.network_id + + gke_support_enabled = var.gke_support_enabled + + timeouts { + create = "1h" + update = "1h" + delete = "1h" + } + + depends_on = [var.private_vpc_connection_peering, data.google_storage_bucket.lustre_import_bucket] + + lifecycle { + precondition { + condition = data.google_compute_network_peering.private_peering.state == "ACTIVE" + error_message = "The subnetwork that the lustre instance is hosted on must have private service access." + } + } + + provisioner "local-exec" { + command = < 0 ]]; then + curl -X POST \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $(gcloud auth print-access-token)" \ + -d '{"gcsPath": {"uri":"${coalesce(var.import_gcs_bucket_uri, "gs://")}"}, "lustrePath": {"path":"${local.destination_path}"}}' \ + https://lustre.googleapis.com/v1/projects/${var.project_id}/locations/${var.zone}/instances/${local.instance_id}:importData + fi + EOF + interpreter = ["bash", "-c"] + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/metadata.yaml new file mode 100644 index 0000000000..66da9827b6 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - lustre.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/outputs.tf new file mode 100644 index 0000000000..fdc0fb20e1 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/outputs.tf @@ -0,0 +1,43 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "network_storage" { + description = "Describes a Managed Lustre instance." + value = { + server_ip = local.server_ip + remote_mount = local.remote_mount + local_mount = var.local_mount + fs_type = local.fs_type + mount_options = local.mount_options + client_install_runner = local.install_managed_lustre_client_runner + mount_runner = local.mount_runner + } +} + +output "install_managed_lustre_client" { + description = "Script for installing Managed Lustre client" + value = file("${path.module}/scripts/install-managed-lustre-client.sh") +} + +output "lustre_id" { + description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}`" + value = google_lustre_instance.lustre_instance.instance_id +} + +output "capacity_gib" { + description = "File share capacity in GiB." + value = google_lustre_instance.lustre_instance.capacity_gib +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh new file mode 100644 index 0000000000..878130ab47 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Install Managed Lustre client modules +# Based on these instructions: https://cloud.google.com/managed-lustre/docs/connect-from-compute-engine + +# The client modules currently only support Rocky 8, and Ubuntu 20.04/22.04 + +set -e + +GKE_ENABLED=$1 + +# Update lnet to enable GKE supported Lustre instance +if [[ $GKE_ENABLED == "1" ]]; then + if [[ -f "/etc/modprobe.d/lnet.conf" ]] && grep -Fq "options lnet accept_port" /etc/modprobe.d/lnet.conf; then + echo "Lnet accept port already set, continuing without updating /etc/modprobe.d/lnet.conf" + else + echo "options lnet accept_port=6988" >>/etc/modprobe.d/lnet.conf + fi +fi + +if grep -q lustre /proc/filesystems; then + echo "Skipping managed lustre client install as it is already supported" + exit 0 +fi + +# Get distro information +. /etc/os-release +DIST="NA" +if [[ $NAME == *"Ubuntu"* ]]; then + if [[ $VERSION_ID == "20.04" || $VERSION_ID == "22.04" ]]; then + DIST="Ubuntu" + fi +elif [[ $NAME == *"Rocky"* ]]; then + if [[ $VERSION_ID == "8"* ]]; then + DIST="Rocky" + fi +fi + +if [[ ${DIST} == "Ubuntu" ]]; then + KEY_LOC=/etc/apt/keyrings + KEY_NAME=gcp-ar-repo.gpg + # Download new repo key + mkdir -p "${KEY_LOC}" + wget -O - https://us-apt.pkg.dev/doc/repo-signing-key.gpg 2>/dev/null | gpg --dearmor - | tee "${KEY_LOC}/${KEY_NAME}" >/dev/null + + # Set up apt repo + echo "deb [ signed-by=${KEY_LOC}/${KEY_NAME} ] https://us-apt.pkg.dev/projects/lustre-client-binaries lustre-client-ubuntu-${UBUNTU_CODENAME} main" | tee -a /etc/apt/sources.list.d/artifact-registry.list + + # Install modules + apt update + apt install -y "lustre-client-modules-$(uname -r)" lustre-client-utils || (echo "Error finding Lustre module packages, Lustre package may not exist for this kernel version" && exit 1) +elif [[ ${DIST} == "Rocky" ]]; then + # Set up yum repo + touch /etc/yum.repos.d/artifact-registry.repo + tee -a /etc/yum.repos.d/artifact-registry.repo <<-EOF + [lustre-client-rocky-8] + name=lustre-client-rocky-8 + baseurl=https://us-yum.pkg.dev/projects/lustre-client-binaries/lustre-client-rocky-8 + enabled=1 + repo_gpgcheck=0 + gpgcheck=0 + EOF + # Install modules + yum makecache + yum --enablerepo=lustre-client-rocky-8 install -y kmod-lustre-client lustre-client +fi + +if [[ $DIST != "NA" ]]; then + # Load the new lustre client module + modprobe lustre +fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh new file mode 100644 index 0000000000..e2509fb4a1 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SERVER_IP=$1 +REMOTE_MOUNT=$2 +LOCAL_MOUNT=$3 +FS_TYPE=$4 +MOUNT_OPTIONS=$5 + +[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" + +if [ "${FS_TYPE}" = "gcsfuse" ]; then + FS_SPEC="${REMOTE_MOUNT}" +else + FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" +fi + +SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" +EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" + +grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false +grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false +findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/variables.tf new file mode 100644 index 0000000000..aa5d3be0ae --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/variables.tf @@ -0,0 +1,131 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "ID of project in which Lustre instance will be created." + type = string +} + +variable "description" { + description = "Description of the created Lustre instance." + type = string + default = "Lustre Instance" +} + +variable "deployment_name" { + description = "Name of the HPC deployment, used as name of the Lustre instance if no name is specified." + type = string +} + +variable "zone" { + description = "Location for the Lustre instance." + type = string +} + +variable "name" { + description = "Name of the Lustre instance" + type = string +} + +variable "network_id" { + description = <<-EOT + The ID of the GCE VPC network to which the instance is connected given in the format: + `projects//global/networks/`" + EOT + type = string + nullable = false + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "network_self_link" { + description = "Network self-link this instance will be on, required for checking private service access" + type = string + nullable = false +} + +variable "remote_mount" { + description = "Remote mount point of the Managed Lustre instance" + type = string + nullable = false +} + +variable "local_mount" { + description = "Local mount point for the Managed Lustre instance." + type = string + default = "/shared" +} + +variable "size_gib" { + description = "Storage size of the Managed Lustre instance in GB. See https://cloud.google.com/managed-lustre/docs/create-instance for limitations" + type = number + default = 18000 +} + +variable "per_unit_storage_throughput" { + description = "Throughput of the instance in MB/s/TiB. Valid values are 125, 250, 500, 1000." + type = number + default = 1000 +} + +variable "labels" { + description = "Labels to add to the Managed Lustre instance. Key-value pairs." + type = map(string) +} + +variable "mount_options" { + description = "Mounting options for the file system." + type = string + default = "defaults,_netdev" +} + +variable "private_vpc_connection_peering" { + description = <<-EOT + The name of the VPC Network peering connection. + If using new VPC, please use community/modules/network/private-service-access to create private-service-access and + If using existing VPC with private-service-access enabled, set this manually." + EOT + type = string + nullable = false +} + +variable "gke_support_enabled" { + description = <<-EOT + Set to true to create Managed Lustre instance with GKE compatibility. + Note: This does not work with Slurm, the Slurm image must be built with + the correct compatibility. + EOT + type = bool + nullable = false + default = false +} + +variable "import_gcs_bucket_uri" { + description = <<-EOT + The name of the GCS bucket to import data from to managed lustre. Data will + be imported to the local_mount directory. Changing this value will not + trigger a redeployment, to prevent data deletion. + EOT + type = string + default = null + + validation { + condition = startswith(coalesce(var.import_gcs_bucket_uri, "gs://"), "gs://") + error_message = "The GCS bucket uri must start with 'gs://'" + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/versions.tf new file mode 100644 index 0000000000..59808977a6 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/versions.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.27.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.57.0" + } + provider_meta "google-beta" { + module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.57.0" + } + + required_version = ">= 1.3.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/README.md b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/README.md new file mode 100644 index 0000000000..0b942f067f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/README.md @@ -0,0 +1,196 @@ +## Description + +This module creates [parallelstore](https://cloud.google.com/parallelstore) +instance. Parallelstore is Google Cloud's first party parallel file system +service based on [Intel DAOS](https://docs.daos.io/v2.2/) + +### Supported Operating Systems + +A parallelstore instance can be used with Slurm cluster or compute +VM running Ubuntu 22.04, debian 12 or HPC Rocky Linux 8. + +### Parallelstore Quota + +To get access to a private preview of Parallelstore APIs, your project needs to +be allowlisted. To set this up, please work with your account representative. + +### Parallelstore mount options + +After parallelstore instance is created, you can specify mount options depending +upon your workload. DAOS is configured to deliver the best user experience for +interactive workloads with aggressive caching. If you are running parallel +workloads concurrently accessing the sane files from multiple client nodes, it +is recommended to disable the writeback cache to avoid cross-client consistency +issues. You can specify different mount options as follows, + +```yaml + - id: parallelstore + source: modules/file-system/parallelstore + use: [network, ps_connect] + settings: + mount_options: "disable-wb-cache,thread-count=20,eq-count=8" +``` + +### Example - New VPC + +For parallelstore instance, Below snippet creates new VPC and configures private-service-access +for this newly created network. + +```yaml + - id: network + source: modules/network/vpc + + # Private Service Access (PSA) requires the compute.networkAdmin role which is + # included in the Owner role, but not Editor. + # PSA is required for all Parallelstore functionality. + # https://cloud.google.com/vpc/docs/configure-private-services-access#permissions + - id: private_service_access + source: community/modules/network/private-service-access + use: [network] + settings: + prefix_length: 24 + + - id: parallelstore + source: modules/file-system/parallelstore + use: [network, private_service_access] +``` + +### Example - Existing VPC + +If you want to use existing network with private-service-access configured, you need +to manually provide `private_vpc_connection_peering` to the parallelstore module. +You can get this details from the Google Cloud Console UI in `VPC network peering` +section. Below is the example of using existing network and creating parallelstore. +If existing network is not configured with private-service-access, you can follow +[Configure private service access](https://cloud.google.com/vpc/docs/configure-private-services-access) +to set it up. + +```yaml + - id: network + source: modules/network/pre-existing-vpc + settings: + network_name: // Add network name + subnetwork_name: // Add subnetwork name + + - id: parallelstore + source: modules/file-system/parallelstore + use: [network] + settings: + private_vpc_connection_peering: # will look like "servicenetworking.googleapis.com" +``` + +### Import data from GCS bucket + +You can import data from your GCS bucket to parallelstore instance. Important to +note that data may not be available to the instance immediately. This depends on +latency and size of data. Below is the example of importing data from bucket. + +```yaml + - id: parallelstore + source: modules/file-system/parallelstore + use: [network] + settings: + import_gcs_bucket_uri: gs://gcs-bucket/folder-path + import_destination_path: /gcs/import/ +``` + +Here you can replace `import_gcs_bucket_uri` with the uri of sub folder within GCS +bucket and `import_destination_path` with local directory within parallelstore +instance. + +### Additional configuration for DAOS agent and dfuse +Use `daos_agent_config` to provide additional configuration for `daos_agent`, for example: + +```yaml +- id: parallelstorefs + source: modules/file-system/pre-existing-network-storage + settings: + daos_agent_config: | + credential_config: + cache_expiration: 1m +``` + +Use `dfuse_environment` to provide additional environment variables for `dfuse` process, for example: + +```yaml +- id: parallelstorefs + source: modules/file-system/parallelstore + settings: + dfuse_environment: + D_LOG_FILE: /tmp/client.log + D_APPEND_PID_TO_LOG: 1 + D_LOG_MASK: debug +``` + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13 | +| [google](#requirement\_google) | >= 6.13.0 | +| [null](#requirement\_null) | ~> 3.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.13.0 | +| [null](#provider\_null) | ~> 3.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_parallelstore_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/parallelstore_instance) | resource | +| [null_resource.hydration](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [daos\_agent\_config](#input\_daos\_agent\_config) | Additional configuration to be added to daos\_config.yml | `string` | `""` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment. | `string` | n/a | yes | +| [dfuse\_environment](#input\_dfuse\_environment) | Additional environment variables for DFuse process | `map(string)` | `{}` | no | +| [directory\_stripe](#input\_directory\_stripe) | The parallelstore stripe level for directories. | `string` | `null` | no | +| [file\_stripe](#input\_file\_stripe) | The parallelstore stripe level for files. | `string` | `null` | no | +| [import\_destination\_path](#input\_import\_destination\_path) | The name of local path to import data on parallelstore instance from GCS bucket. | `string` | `null` | no | +| [import\_gcs\_bucket\_uri](#input\_import\_gcs\_bucket\_uri) | The name of the GCS bucket to import data from to parallelstore. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to parallel store instance. | `map(string)` | `{}` | no | +| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/parallelstore"` | no | +| [mount\_options](#input\_mount\_options) | Options describing various aspects of the parallelstore instance. | `string` | `"disable-wb-cache,thread-count=16,eq-count=8"` | no | +| [name](#input\_name) | Name of parallelstore instance. | `string` | `null` | no | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | +| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually." | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [size\_gb](#input\_size\_gb) | Storage size of the parallelstore instance in GB. | `number` | `12000` | no | +| [zone](#input\_zone) | Location for parallelstore instance. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Instructions to monitor import-data operation from GCS bucket to parallelstore. | +| [network\_storage](#output\_network\_storage) | Describes a parallelstore instance. | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/main.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/main.tf new file mode 100644 index 0000000000..acc2a0551e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/main.tf @@ -0,0 +1,74 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "parallelstore", ghpc_role = "file-system" }) +} + +locals { + fs_type = "daos" + server_ip = "" + remote_mount = "" + id = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" + access_points = jsonencode(google_parallelstore_instance.instance.access_points) + destination_path = var.import_destination_path == null ? "/" : var.import_destination_path + + client_install_runner = { + "type" = "shell" + "source" = "${path.module}/scripts/install-daos-client.sh" + "destination" = "install_daos_client.sh" + } + + mount_runner = { + "type" = "shell" + "content" = templatefile("${path.module}/templates/mount-daos.sh.tftpl", { + access_points = local.access_points + daos_agent_config = var.daos_agent_config + dfuse_environment = var.dfuse_environment + local_mount = var.local_mount + mount_options = join(" ", [for opt in split(",", var.mount_options) : "--${opt}"]) + }) + "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" + } +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_parallelstore_instance" "instance" { + project = var.project_id + instance_id = local.id + location = var.zone + capacity_gib = var.size_gb + network = var.network_id + file_stripe_level = var.file_stripe + directory_stripe_level = var.directory_stripe + + labels = local.labels + + depends_on = [var.private_vpc_connection_peering] +} + +resource "null_resource" "hydration" { + count = var.import_gcs_bucket_uri != null ? 1 : 0 + + depends_on = [resource.google_parallelstore_instance.instance] + provisioner "local-exec" { + command = "curl -X POST -H \"Content-Type: application/json\" -H \"Authorization: Bearer $(gcloud auth print-access-token)\" -d '{\"source_gcs_bucket\": {\"uri\":\"${var.import_gcs_bucket_uri}\"}, \"destination_parallelstore\": {\"path\":\"${local.destination_path}\"}}' https://parallelstore.googleapis.com/v1beta/projects/${var.project_id}/locations/${var.zone}/instances/${local.id}:importData" + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/metadata.yaml new file mode 100644 index 0000000000..c0994d15bb --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - parallelstore.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/outputs.tf new file mode 100644 index 0000000000..f6e817ac8a --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/outputs.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + operation_instructions = <<-EOT + Data is being imported from GCS bucket to parallelstore instance. It may + not be available immediately. + EOT +} + +output "network_storage" { + description = "Describes a parallelstore instance." + value = { + server_ip = local.server_ip + remote_mount = local.remote_mount + local_mount = var.local_mount + fs_type = local.fs_type + mount_options = var.mount_options + client_install_runner = local.client_install_runner + mount_runner = local.mount_runner + } + + precondition { + condition = var.import_gcs_bucket_uri != null || var.import_destination_path == null + error_message = <<-EOD + Please specify import_gcs_bucket_uri to import data to parallelstore instance. + EOD + } +} + +output "instructions" { + description = "Instructions to monitor import-data operation from GCS bucket to parallelstore." + value = var.import_gcs_bucket_uri != null ? local.operation_instructions : "Data is not imported from GCS bucket." +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh new file mode 100644 index 0000000000..e96eadb56a --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') + +if ! { + { [[ "${OS_ID}" = "rocky" ]] || [[ "${OS_ID}" = "rhel" ]]; } && { [[ "${OS_VERSION_MAJOR}" = "8" ]] || [[ "${OS_VERSION_MAJOR}" = "9" ]]; } || + { [[ "${OS_ID}" = "ubuntu" ]] && [[ "${OS_VERSION}" = "22.04" ]]; } || + { [[ "${OS_ID}" = "debian" ]] && [[ "${OS_VERSION_MAJOR}" = "12" ]]; } +}; then + echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." + exit 1 +fi + +if [ -x /bin/daos ]; then + echo "DAOS already installed" + daos version +else + # Install the DAOS client library + # The following commands should be executed on each client vm. + ## For Rocky linux 8 / RedHat 8. + if [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; then + # 1) Add the Parallelstore package repository + cat >/etc/yum.repos.d/parallelstore-v2-6-el"${OS_VERSION_MAJOR}".repo <<-EOF + [parallelstore-v2-6-el${OS_VERSION_MAJOR}] + name=Parallelstore EL${OS_VERSION_MAJOR} v2.6 + baseurl=https://us-central1-yum.pkg.dev/projects/parallelstore-packages/v2-6-el${OS_VERSION_MAJOR} + enabled=1 + repo_gpgcheck=0 + gpgcheck=0 + EOF + + ## TODO: Remove disable automatic update script after issue is fixed. + if [ -x /usr/bin/google_disable_automatic_updates ]; then + /usr/bin/google_disable_automatic_updates + fi + dnf clean all + dnf makecache + + # 2) Install daos-client + dnf install -y epel-release # needed for capstone + dnf install -y daos-client + + # 3) Upgrade libfabric + dnf upgrade -y libfabric + + # For Ubuntu 22.04 and debian 12, + elif [[ "${OS_ID}" = "ubuntu" ]] || [[ "${OS_ID}" = "debian" ]]; then + # shellcheck disable=SC2034 + DEBIAN_FRONTEND=noninteractive + + # 1) Add the Parallelstore package repository + curl -o /etc/apt/trusted.gpg.d/us-central1-apt.pkg.dev.asc https://us-central1-apt.pkg.dev/doc/repo-signing-key.gpg + echo "deb https://us-central1-apt.pkg.dev/projects/parallelstore-packages v2-6-deb main" >/etc/apt/sources.list.d/artifact-registry.list + + apt-get update + + # 2) Install daos-client + apt-get install -y daos-client + + # 3) Create daos_agent.service (comes pre-installed with RedHat) + if ! getent passwd daos_agent >/dev/null 2>&1; then + useradd daos_agent + fi + cat >/etc/systemd/system/daos_agent.service <<-EOF + [Unit] + Description=DAOS Agent + StartLimitIntervalSec=60 + Wants=network-online.target + After=network-online.target + + [Service] + Type=notify + User=daos_agent + Group=daos_agent + RuntimeDirectory=daos_agent + RuntimeDirectoryMode=0755 + ExecStart=/usr/bin/daos_agent -o /etc/daos/daos_agent.yml + StandardOutput=journal + StandardError=journal + Restart=always + RestartSec=10 + LimitMEMLOCK=infinity + LimitCORE=infinity + StartLimitBurst=5 + + [Install] + WantedBy=multi-user.target + EOF + else + echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." + exit 1 + fi +fi + +exit 0 diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl new file mode 100644 index 0000000000..c6f5d53660 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl @@ -0,0 +1,110 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') + +if ! { + { [[ "$${OS_ID}" = "rocky" ]] || [[ "$${OS_ID}" = "rhel" ]]; } && { [[ "$${OS_VERSION_MAJOR}" = "8" ]] || [[ "$${OS_VERSION_MAJOR}" = "9" ]]; } || + { [[ "$${OS_ID}" = "ubuntu" ]] && [[ "$${OS_VERSION}" = "22.04" ]]; } || + { [[ "$${OS_ID}" = "debian" ]] && [[ "$${OS_VERSION_MAJOR}" = "12" ]]; } +}; then + echo "Unsupported operating system $${OS_ID} $${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." + exit 1 + +fi + +# Edit agent config +daos_config=/etc/daos/daos_agent.yml + +# rewrite $daos_config from scratch +mv $${daos_config} $${daos_config}.orig + +exclude_fabric_ifaces="" +# Get names of network interfaces not in first PCI slot +# The first PCI slot is a standard network adapter while remaining interfaces +# are typically network cards dedicated to GPU or workload communication +if [[ "$${OS_ID}" == "debian" ]] || [[ "$${OS_ID}" = "ubuntu" ]]; then + extra_interfaces=$(find /sys/class/net/ -not -name 'enp0s*' -regextype posix-extended -regex '.*/enp[0-9]+s.*' -printf '"%f"\n' | paste -s -d ',') +elif [[ "$${OS_ID}" = "rocky" ]] || [[ "$${OS_ID}" = "rhel" ]]; then + extra_interfaces=$(find /sys/class/net/ -not -name eth0 -regextype posix-extended -regex '.*/eth[0-9]+' -printf '"%f"\n' | paste -s -d ',') +fi + +cat > $daos_config </etc/systemd/system/"$${service_name}" </global/networks/`" + EOT + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "import_gcs_bucket_uri" { + description = "The name of the GCS bucket to import data from to parallelstore." + type = string + default = null +} + +variable "import_destination_path" { + description = "The name of local path to import data on parallelstore instance from GCS bucket." + type = string + default = null +} + +variable "file_stripe" { + description = "The parallelstore stripe level for files." + type = string + default = null + validation { + condition = var.file_stripe == null ? true : contains([ + "FILE_STRIPE_LEVEL_UNSPECIFIED", + "FILE_STRIPE_LEVEL_MIN", + "FILE_STRIPE_LEVEL_BALANCED", + "FILE_STRIPE_LEVEL_MAX", + ], var.file_stripe) + error_message = "var.file_stripe must be set to \"FILE_STRIPE_LEVEL_UNSPECIFIED\", \"FILE_STRIPE_LEVEL_MIN\", \"FILE_STRIPE_LEVEL_BALANCED\", or \"FILE_STRIPE_LEVEL_MAX\"" + } +} + +variable "directory_stripe" { + description = "The parallelstore stripe level for directories." + type = string + default = null + validation { + condition = var.directory_stripe == null ? true : contains([ + "DIRECTORY_STRIPE_LEVEL_UNSPECIFIED", + "DIRECTORY_STRIPE_LEVEL_MIN", + "DIRECTORY_STRIPE_LEVEL_BALANCED", + "DIRECTORY_STRIPE_LEVEL_MAX", + ], var.directory_stripe) + error_message = "var.directory_stripe must be set to \"DIRECTORY_STRIPE_LEVEL_UNSPECIFIED\", \"DIRECTORY_STRIPE_LEVEL_MIN\", \"DIRECTORY_STRIPE_LEVEL_BALANCED\", or \"DIRECTORY_STRIPE_LEVEL_MAX\"" + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/versions.tf new file mode 100644 index 0000000000..174b5281e4 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/versions.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.13.0" + } + + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/README.md b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/README.md new file mode 100644 index 0000000000..1968fe4c84 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/README.md @@ -0,0 +1,192 @@ +## Description + +This module defines a file-system that already exists (i.e. it does not create +a new file system) in a way that can be shared with other modules. This allows +a compute VM to mount a filesystem that is not part of the current deployment +group. + +The pre-existing network storage can be referenced in the same way as any Cluster +Toolkit supported file-system such as [filestore](../filestore/README.md). + +For more information on network storage options in the Cluster Toolkit, see +the extended [Network Storage documentation](../../../../docs/network_storage.md). + +### Example + +```yaml +- id: homefs + source: modules/file-system/pre-existing-network-storage + settings: + server_ip: ## Set server IP here ## + remote_mount: nfsshare + local_mount: /home + fs_type: nfs +``` + +This creates a pre-existing-network-storage module in terraform at the +provided IP in `server_ip` of type nfs that will be mounted at `/home`. Note +that the `server_ip` must be known before deployment. + +The following is an example of using `pre-existing-network-storage` with a GCS +bucket: + +```yaml +- id: data-bucket + source: modules/file-system/pre-existing-network-storage + settings: + remote_mount: my-bucket-name + local_mount: /data + fs_type: gcsfuse + mount_options: defaults,_netdev,implicit_dirs +``` + +The `implicit_dirs` mount option allows object paths to be treated as if they +were directories. This is important when working with files that were created by +another source, but there may have performance impacts. The `_netdev` mount option +denotes that the storage device requires network access. + +The following is an example of using `pre-existing-network-storage` with the `lustre` +filesystem: + +```yaml +- id: lustrefs + source: modules/file-system/pre-existing-network-storage + settings: + fs_type: lustre + server_ip: 192.168.227.11@tcp + local_mount: /scratch + remote_mount: /exacloud +``` + +Note the use of the MGS NID (Network ID) in the `server_ip` field - in +particular, note the `@tcp` suffix. + +The following is an example of using `pre-existing-network-storage` with the +`managed_lustre` filesystem: + +```yaml +- id: lustrefs + source: modules/file-system/pre-existing-network-storage + settings: + fs_type: managed_lustre + server_ip: 192.168.227.11@tcp + local_mount: /scratch + remote_mount: /mg_lustre +``` + +This is similar to the `lustre` filesystem, with the exception that it connects +with a managed Lustre instance hosted by GCP. Currently only Rocky 8 and +Ubuntu 20.04 and Ubuntu 22.04 are supported. + +The following is an example of using `pre-existing-network-storage` with the `daos` +filesystem. In order to use existing `parallelstore` instance, `fs_type` needs to be +explicitly mentioned in blueprint. The `remote_mount` option refers to `access_points` +for `parallelstore` instance. + +```yaml +- id: parallelstorefs + source: modules/file-system/pre-existing-network-storage + settings: + fs_type: daos + remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" + mount_options: disable-wb-cache,thread-count=16,eq-count=8 +``` + +Parallelstore supports additional options for its mountpoints under `parallelstore_options` setting. +Use `daos_agent_config` to provide additional configuration for `daos_agent`, for example: + +```yaml +- id: parallelstorefs + source: modules/file-system/pre-existing-network-storage + settings: + fs_type: daos + remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" + mount_options: disable-wb-cache,thread-count=16,eq-count=8 + parallelstore_options: + daos_agent_config: | + credential_config: + cache_expiration: 1m +``` + +Use `dfuse_environment` to provide additional environment variables for `dfuse` process, for example: + +```yaml +- id: parallelstorefs + source: modules/file-system/pre-existing-network-storage + settings: + fs_type: daos + remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" + mount_options: disable-wb-cache,thread-count=16,eq-count=8 + parallelstore_options: + dfuse_environment: + D_LOG_FILE: /tmp/client.log + D_APPEND_PID_TO_LOG: 1 + D_LOG_MASK: debug +``` + +### Mounting + +For the `fs_type` listed below, this module will provide `client_install_runner` +and `mount_runner` outputs. These can be used to create a startup script to +mount the network storage system. + +Supported `fs_type`: + +- nfs +- lustre +- managed_lustre +- gcsfuse +- daos + +[scripts/mount.sh](./scripts/mount.sh) is used as the contents of +`mount_runner`. This script will update `/etc/fstab` and mount the network +storage. This script will fail if the specified `local_mount` is already being +used by another entry in `/etc/fstab`. + +Both of these steps are automatically handled with the use of the `use` command +in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in +the network storage doc for a complete list of supported modules. + +[matrix]: ../../../docs/network_storage.md#compatibility-matrix + +## License + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [fs\_type](#input\_fs\_type) | Type of file system to be mounted (e.g., nfs, lustre) | `string` | `"nfs"` | no | +| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | +| [managed\_lustre\_options](#input\_managed\_lustre\_options) | Managed Lustre specific options:
gke\_support\_enabled (bool, default = false)
Note: gke\_support\_enabled does not work with Slurm, the Slurm image must be built with
the correct compatibility. | 
object({
    gke_support_enabled = optional(bool, false)
  })
| `{}` | no | +| [mount\_options](#input\_mount\_options) | Options describing various aspects of the file system. Consider adding setting to 'defaults,\_netdev,implicit\_dirs' when using gcsfuse. | `string` | `"defaults,_netdev"` | no | +| [parallelstore\_options](#input\_parallelstore\_options) | Parallelstore specific options | 
object({
    daos_agent_config = optional(string, "")
    dfuse_environment = optional(map(string), {})
  })
| `{}` | no | +| [remote\_mount](#input\_remote\_mount) | Remote FS name or export. This is the exported directory for nfs, fs name for lustre, and bucket name (without gs://) for gcsfuse. | `string` | n/a | yes | +| [server\_ip](#input\_server\_ip) | The device name as supplied to fs-tab, excluding remote fs-name(for nfs, that is the server IP, for lustre [:]). This can be omitted for gcsfuse. | `string` | `""` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [client\_install\_runner](#output\_client\_install\_runner) | Runner that performs client installation needed to use file system. | +| [mount\_runner](#output\_mount\_runner) | Runner that mounts the file system. | +| [network\_storage](#output\_network\_storage) | Describes a remote network storage to be mounted by fs-tab. | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml new file mode 100644 index 0000000000..641832182d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: [] diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf new file mode 100644 index 0000000000..203b6dfdac --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf @@ -0,0 +1,124 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "network_storage" { + description = "Describes a remote network storage to be mounted by fs-tab." + value = { + server_ip = var.server_ip + remote_mount = local.remote_mount + local_mount = var.local_mount + fs_type = local.fs_type + mount_options = var.mount_options + client_install_runner = local.client_install_runner + mount_runner = local.mount_runner + } +} + +locals { + # Update remote mount to include a slash if the fs_type requires one to exist + remote_mount_with_slash = length(regexall("^/.*", var.remote_mount)) > 0 ? ( + var.remote_mount + ) : format("/%s", var.remote_mount) + remote_mount = contains(local.mount_vanilla_supported_fstype, local.fs_type) ? ( + local.remote_mount_with_slash + ) : var.remote_mount + + ml_gke_support_enabled = coalesce(try(var.managed_lustre_options.gke_support_enabled, false), false) + + # Collapse fs_type lustre and managed lustre for most uses, only needs to be + # different for client installation + fs_type = strcontains(var.fs_type, "lustre") ? "lustre" : var.fs_type + + # Client Install + ddn_lustre_client_install_script = templatefile( + "${path.module}/templates/ddn_exascaler_luster_client_install.tftpl", + { + server_ip = split("@", var.server_ip)[0] + remote_mount = local.remote_mount + local_mount = var.local_mount + } + ) + managed_lustre_client_install_script = file("${path.module}/scripts/install-managed-lustre-client.sh") + nfs_client_install_script = file("${path.module}/scripts/install-nfs-client.sh") + gcs_fuse_install_script = file("${path.module}/scripts/install-gcs-fuse.sh") + daos_client_install_script = file("${path.module}/scripts/install-daos-client.sh") + + install_scripts = { + "lustre" = local.ddn_lustre_client_install_script + "managed_lustre" = local.managed_lustre_client_install_script + "nfs" = local.nfs_client_install_script + "gcsfuse" = local.gcs_fuse_install_script + "daos" = local.daos_client_install_script + } + + client_install_runner = { + "type" = "shell" + "content" = lookup(local.install_scripts, var.fs_type, "echo 'skipping: client_install_runner not yet supported for ${var.fs_type}'") + "destination" = "install_filesystem_client${replace(var.local_mount, "/", "_")}.sh" + "args" = local.ml_gke_support_enabled ? "1" : "" + } + + mount_vanilla_supported_fstype = ["lustre", "nfs"] + mount_runner_vanilla = { + "type" = "shell" + "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" + "args" = "\"${var.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${var.mount_options}\"" + "content" = ( + contains(local.mount_vanilla_supported_fstype, local.fs_type) ? + file("${path.module}/scripts/mount.sh") : + "echo 'skipping: mount_runner not yet supported for ${var.fs_type}'" + ) + } + gcsbucket = trimprefix(var.remote_mount, "gs://") + mount_runner_gcsfuse = { + "type" = "shell" + "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" + "args" = "\"not-used\" \"${local.gcsbucket}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${var.mount_options}\"" + "content" = file("${path.module}/scripts/mount.sh") + } + + mount_runner_daos = { + "type" = "shell" + "content" = templatefile("${path.module}/templates/mount-daos.sh.tftpl", { + access_points = var.remote_mount + daos_agent_config = var.parallelstore_options.daos_agent_config + dfuse_environment = var.parallelstore_options.dfuse_environment + local_mount = var.local_mount + # avoid passing "--" as mount option to dfuse + mount_options = length(var.mount_options) == 0 ? "" : join(" ", [for opt in split(",", var.mount_options) : "--${opt}"]) + }) + "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" + } + + mount_scripts = { + "lustre" = local.mount_runner_vanilla + "nfs" = local.mount_runner_vanilla + "gcsfuse" = local.mount_runner_gcsfuse + "daos" = local.mount_runner_daos + } + + mount_runner = lookup(local.mount_scripts, local.fs_type, local.mount_runner_vanilla) +} + +output "client_install_runner" { + description = "Runner that performs client installation needed to use file system." + value = local.client_install_runner +} + +output "mount_runner" { + description = "Runner that mounts the file system." + value = local.mount_runner +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh new file mode 100644 index 0000000000..e96eadb56a --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') +OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') + +if ! { + { [[ "${OS_ID}" = "rocky" ]] || [[ "${OS_ID}" = "rhel" ]]; } && { [[ "${OS_VERSION_MAJOR}" = "8" ]] || [[ "${OS_VERSION_MAJOR}" = "9" ]]; } || + { [[ "${OS_ID}" = "ubuntu" ]] && [[ "${OS_VERSION}" = "22.04" ]]; } || + { [[ "${OS_ID}" = "debian" ]] && [[ "${OS_VERSION_MAJOR}" = "12" ]]; } +}; then + echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." + exit 1 +fi + +if [ -x /bin/daos ]; then + echo "DAOS already installed" + daos version +else + # Install the DAOS client library + # The following commands should be executed on each client vm. + ## For Rocky linux 8 / RedHat 8. + if [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; then + # 1) Add the Parallelstore package repository + cat >/etc/yum.repos.d/parallelstore-v2-6-el"${OS_VERSION_MAJOR}".repo <<-EOF + [parallelstore-v2-6-el${OS_VERSION_MAJOR}] + name=Parallelstore EL${OS_VERSION_MAJOR} v2.6 + baseurl=https://us-central1-yum.pkg.dev/projects/parallelstore-packages/v2-6-el${OS_VERSION_MAJOR} + enabled=1 + repo_gpgcheck=0 + gpgcheck=0 + EOF + + ## TODO: Remove disable automatic update script after issue is fixed. + if [ -x /usr/bin/google_disable_automatic_updates ]; then + /usr/bin/google_disable_automatic_updates + fi + dnf clean all + dnf makecache + + # 2) Install daos-client + dnf install -y epel-release # needed for capstone + dnf install -y daos-client + + # 3) Upgrade libfabric + dnf upgrade -y libfabric + + # For Ubuntu 22.04 and debian 12, + elif [[ "${OS_ID}" = "ubuntu" ]] || [[ "${OS_ID}" = "debian" ]]; then + # shellcheck disable=SC2034 + DEBIAN_FRONTEND=noninteractive + + # 1) Add the Parallelstore package repository + curl -o /etc/apt/trusted.gpg.d/us-central1-apt.pkg.dev.asc https://us-central1-apt.pkg.dev/doc/repo-signing-key.gpg + echo "deb https://us-central1-apt.pkg.dev/projects/parallelstore-packages v2-6-deb main" >/etc/apt/sources.list.d/artifact-registry.list + + apt-get update + + # 2) Install daos-client + apt-get install -y daos-client + + # 3) Create daos_agent.service (comes pre-installed with RedHat) + if ! getent passwd daos_agent >/dev/null 2>&1; then + useradd daos_agent + fi + cat >/etc/systemd/system/daos_agent.service <<-EOF + [Unit] + Description=DAOS Agent + StartLimitIntervalSec=60 + Wants=network-online.target + After=network-online.target + + [Service] + Type=notify + User=daos_agent + Group=daos_agent + RuntimeDirectory=daos_agent + RuntimeDirectoryMode=0755 + ExecStart=/usr/bin/daos_agent -o /etc/daos/daos_agent.yml + StandardOutput=journal + StandardError=journal + Restart=always + RestartSec=10 + LimitMEMLOCK=infinity + LimitCORE=infinity + StartLimitBurst=5 + + [Install] + WantedBy=multi-user.target + EOF + else + echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." + exit 1 + fi +fi + +exit 0 diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh new file mode 100644 index 0000000000..f8a990260b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +if [ ! "$(which gcsfuse)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ]; then + tee /etc/yum.repos.d/gcsfuse.repo >/dev/null <>/etc/modprobe.d/lnet.conf + fi +fi + +if grep -q lustre /proc/filesystems; then + echo "Skipping managed lustre client install as it is already supported" + exit 0 +fi + +# Get distro information +. /etc/os-release +DIST="NA" +if [[ $NAME == *"Ubuntu"* ]]; then + if [[ $VERSION_ID == "20.04" || $VERSION_ID == "22.04" ]]; then + DIST="Ubuntu" + fi +elif [[ $NAME == *"Rocky"* ]]; then + if [[ $VERSION_ID == "8"* ]]; then + DIST="Rocky" + fi +fi + +if [[ ${DIST} == "Ubuntu" ]]; then + KEY_LOC=/etc/apt/keyrings + KEY_NAME=gcp-ar-repo.gpg + # Download new repo key + mkdir -p "${KEY_LOC}" + wget -O - https://us-apt.pkg.dev/doc/repo-signing-key.gpg 2>/dev/null | gpg --dearmor - | tee "${KEY_LOC}/${KEY_NAME}" >/dev/null + + # Set up apt repo + echo "deb [ signed-by=${KEY_LOC}/${KEY_NAME} ] https://us-apt.pkg.dev/projects/lustre-client-binaries lustre-client-ubuntu-${UBUNTU_CODENAME} main" | tee -a /etc/apt/sources.list.d/artifact-registry.list + + # Install modules + apt update + apt install -y "lustre-client-modules-$(uname -r)" lustre-client-utils || (echo "Error finding Lustre module packages, Lustre package may not exist for this kernel version" && exit 1) +elif [[ ${DIST} == "Rocky" ]]; then + # Set up yum repo + touch /etc/yum.repos.d/artifact-registry.repo + tee -a /etc/yum.repos.d/artifact-registry.repo <<-EOF + [lustre-client-rocky-8] + name=lustre-client-rocky-8 + baseurl=https://us-yum.pkg.dev/projects/lustre-client-binaries/lustre-client-rocky-8 + enabled=1 + repo_gpgcheck=0 + gpgcheck=0 + EOF + # Install modules + yum makecache + yum --enablerepo=lustre-client-rocky-8 install -y kmod-lustre-client lustre-client +fi + +if [[ $DIST != "NA" ]]; then + # Load the new lustre client module + modprobe lustre +fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh new file mode 100644 index 0000000000..9f842c5d7c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! "$(which mount.nfs)" ]; then + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || + [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + major_version=$(rpm -E "%{rhel}") + enable_repo="" + if [ "${major_version}" -eq "7" ]; then + enable_repo="base,epel" + elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then + enable_repo="baseos" + else + echo "Unsupported version of centos/RHEL/Rocky" + return 1 + fi + yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils + elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get -y install nfs-common + else + echo 'Unsuported distribution' + return 1 + fi +fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh new file mode 100644 index 0000000000..e2509fb4a1 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +SERVER_IP=$1 +REMOTE_MOUNT=$2 +LOCAL_MOUNT=$3 +FS_TYPE=$4 +MOUNT_OPTIONS=$5 + +[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" + +if [ "${FS_TYPE}" = "gcsfuse" ]; then + FS_SPEC="${REMOTE_MOUNT}" +else + FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" +fi + +SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" +EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" + +grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false +grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false +findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false + +# Do nothing and success if exact entry is already in fstab and mounted +if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then + echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" + exit 0 +fi + +# Fail if previous fstab entry is using same local mount +if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" + exit 1 +fi + +# Add to fstab if entry is not already there +if [ "${EXACT_IN_FSTAB}" = false ]; then + echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" + echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab +fi + +# Mount from fstab +echo "Mounting --target ${LOCAL_MOUNT} from fstab" +mkdir -p "${LOCAL_MOUNT}" +mount --target "${LOCAL_MOUNT}" diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl new file mode 100644 index 0000000000..f5f0291e85 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl @@ -0,0 +1,50 @@ +#!/bin/sh + +# Copyright 2022 DataDirect Networks +# Modifications Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Prior Art: https://github.com/DDNStorage/exascaler-cloud-terraform/blob/78deadbb2c1fa7e4603cf9605b0f7d1782117954/gcp/templates/client-script.tftpl + +# install new EXAScaler Cloud clients: +# all instances must be in the same zone +# and connected to the same network and subnet +# to set up EXAScaler Cloud filesystem on a new client instance, +# run the following commands on the client with root privileges: +set -e +if [[ ! -z $(cat /proc/filesystems | grep lustre) ]]; then + echo "Skipping lustre client install as it is already supported" + exit 0 +fi + +cat >/etc/esc-client.conf< $daos_config </etc/systemd/system/"$${service_name}" < +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string
    count = number
    gpu_driver_installation_config = optional(object({
      gpu_driver_version = string
    }), { gpu_driver_version = "DEFAULT" })
    gpu_partition_size = optional(string)
    gpu_sharing_config = optional(object({
      gpu_sharing_strategy       = string
      max_shared_clients_per_gpu = number
    }))
  }))
| `[]` | no | +| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [guest\_accelerator](#output\_guest\_accelerator) | Sanitized list of the type and count of accelerator cards attached to the instance. | +| [machine\_type\_guest\_accelerator](#output\_machine\_type\_guest\_accelerator) | List of the type and count of accelerator cards attached to the specified machine type. | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/internal/gpu-definition/main.tf b/vdi-test-scott/primary/modules/embedded/modules/internal/gpu-definition/main.tf new file mode 100644 index 0000000000..d85a9f8eb6 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/internal/gpu-definition/main.tf @@ -0,0 +1,96 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "machine_type" { + description = "Machine type to use for the instance creation" + type = string +} + +variable "guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the instance." + type = list(object({ + type = string + count = number + gpu_driver_installation_config = optional(object({ + gpu_driver_version = string + }), { gpu_driver_version = "DEFAULT" }) + gpu_partition_size = optional(string) + gpu_sharing_config = optional(object({ + gpu_sharing_strategy = string + max_shared_clients_per_gpu = number + })) + })) + default = [] + nullable = false +} + +locals { + # example state; terraform will ignore diffs if last element of URL matches + # guest_accelerator = [ + # { + # count = 1 + # type = "https://www.googleapis.com/compute/beta/projects/PROJECT/zones/ZONE/acceleratorTypes/nvidia-tesla-a100" + # }, + # ] + accelerator_machines = { + "a2-highgpu-1g" = { type = "nvidia-tesla-a100", count = 1 }, + "a2-highgpu-2g" = { type = "nvidia-tesla-a100", count = 2 }, + "a2-highgpu-4g" = { type = "nvidia-tesla-a100", count = 4 }, + "a2-highgpu-8g" = { type = "nvidia-tesla-a100", count = 8 }, + "a2-megagpu-16g" = { type = "nvidia-tesla-a100", count = 16 }, + "a2-ultragpu-1g" = { type = "nvidia-a100-80gb", count = 1 }, + "a2-ultragpu-2g" = { type = "nvidia-a100-80gb", count = 2 }, + "a2-ultragpu-4g" = { type = "nvidia-a100-80gb", count = 4 }, + "a2-ultragpu-8g" = { type = "nvidia-a100-80gb", count = 8 }, + "a3-highgpu-1g" = { type = "nvidia-h100-80gb", count = 1 }, + "a3-highgpu-2g" = { type = "nvidia-h100-80gb", count = 2 }, + "a3-highgpu-4g" = { type = "nvidia-h100-80gb", count = 4 }, + "a3-highgpu-8g" = { type = "nvidia-h100-80gb", count = 8 }, + "a3-megagpu-8g" = { type = "nvidia-h100-mega-80gb", count = 8 }, + "a3-ultragpu-8g" = { type = "nvidia-h200-141gb", count = 8 }, + "a4-highgpu-8g-lowmem" = { type = "nvidia-b200", count = 8 }, + "a4-highgpu-8g" = { type = "nvidia-b200", count = 8 }, + "g2-standard-4" = { type = "nvidia-l4", count = 1 }, + "g2-standard-8" = { type = "nvidia-l4", count = 1 }, + "g2-standard-12" = { type = "nvidia-l4", count = 1 }, + "g2-standard-16" = { type = "nvidia-l4", count = 1 }, + "g2-standard-24" = { type = "nvidia-l4", count = 2 }, + "g2-standard-32" = { type = "nvidia-l4", count = 1 }, + "g2-standard-48" = { type = "nvidia-l4", count = 4 }, + "g2-standard-96" = { type = "nvidia-l4", count = 8 }, + } + generated_guest_accelerator = try([local.accelerator_machines[var.machine_type]], []) + + # Select in priority order: + # (1) var.guest_accelerator if not empty + # (2) local.generated_guest_accelerator if not empty + # (3) default to empty list if both are empty + guest_accelerator = try(coalescelist(var.guest_accelerator, local.generated_guest_accelerator), []) +} + +output "guest_accelerator" { + description = "Sanitized list of the type and count of accelerator cards attached to the instance." + value = local.guest_accelerator +} + +output "machine_type_guest_accelerator" { + description = "List of the type and count of accelerator cards attached to the specified machine type." + value = local.generated_guest_accelerator +} + +terraform { + required_version = ">= 1.3" +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/README.md b/vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/README.md new file mode 100644 index 0000000000..8aa9270a0a --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/README.md @@ -0,0 +1,54 @@ + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15.0 | +| [google-beta](#requirement\_google-beta) | >= 6.0.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google-beta](#provider\_google-beta) | >= 6.0.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_network_attachment.self](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_network_attachment) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [connection\_preference](#input\_connection\_preference) | The connection preference of service attachment. | `string` | `"ACCEPT_AUTOMATIC"` | no | +| [name](#input\_name) | Name of the resource. Provided by the client when the resource is created | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | The ID of the project in which the resource belongs. | `string` | n/a | yes | +| [region](#input\_region) | Region where the network attachment resides | `string` | n/a | yes | +| [subnetwork\_self\_links](#input\_subnetwork\_self\_links) | An array of selfLinks of subnets to use for endpoints in the producers that connect to this network attachment. | `list(string)` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [self\_link](#output\_self\_link) | Server-defined URL for the resource. | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/main.tf b/vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/main.tf new file mode 100644 index 0000000000..bbbece7085 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/main.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +variable "connection_preference" { + type = string + description = "The connection preference of service attachment." + default = "ACCEPT_AUTOMATIC" +} + +variable "subnetwork_self_links" { + type = list(string) + description = " An array of selfLinks of subnets to use for endpoints in the producers that connect to this network attachment." +} + +variable "name" { + type = string + description = "Name of the resource. Provided by the client when the resource is created" +} + +variable "project_id" { + type = string + description = "The ID of the project in which the resource belongs." +} + +variable "region" { + type = string + description = "Region where the network attachment resides" +} + + +resource "google_compute_network_attachment" "self" { + provider = google-beta + + project = var.project_id + region = var.region + name = var.name + connection_preference = var.connection_preference + subnetworks = var.subnetwork_self_links +} + + +output "self_link" { + value = google_compute_network_attachment.self.self_link + description = "Server-defined URL for the resource." +} + +terraform { + required_version = ">= 0.15.0" + + required_providers { + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.0.0" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/metadata.yaml new file mode 100644 index 0000000000..e80fc96b9c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/README.md b/vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/README.md new file mode 100644 index 0000000000..aefac9d187 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/README.md @@ -0,0 +1,56 @@ + +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15.0 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_network_peering.peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network_peering) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [export\_custom\_routes](#input\_export\_custom\_routes) | (Optional) Whether to export the custom routes to the peer network. Defaults to false. | `bool` | `null` | no | +| [import\_custom\_routes](#input\_import\_custom\_routes) | (Optional) Whether to import the custom routes from the peer network. Defaults to false. | `bool` | `null` | no | +| [import\_subnet\_routes\_with\_public\_ip](#input\_import\_subnet\_routes\_with\_public\_ip) | (Optional) Whether subnet routes with public IP range are imported. | `bool` | `null` | no | +| [name](#input\_name) | Name of the peering. | `string` | n/a | yes | +| [network\_self\_link](#input\_network\_self\_link) | The primary network of the peering. | `string` | n/a | yes | +| [peer\_network\_self\_link](#input\_peer\_network\_self\_link) | The peer network in the peering. The peer network may belong to a different project. | `string` | n/a | yes | +| [stack\_type](#input\_stack\_type) | (Optional) Which IP version(s) of traffic and routes are allowed to be imported or exported between peer networks. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [peering\_name](#output\_peering\_name) | Name of the peering. | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/main.tf b/vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/main.tf new file mode 100644 index 0000000000..386fa9377b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/main.tf @@ -0,0 +1,80 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "name" { + type = string + description = "Name of the peering." +} + +variable "network_self_link" { + type = string + description = "The primary network of the peering." +} + +variable "peer_network_self_link" { + type = string + description = "The peer network in the peering. The peer network may belong to a different project." +} + +variable "export_custom_routes" { + type = bool + description = "(Optional) Whether to export the custom routes to the peer network. Defaults to false." + default = null +} + +variable "import_custom_routes" { + type = bool + description = "(Optional) Whether to import the custom routes from the peer network. Defaults to false." + default = null +} + +variable "import_subnet_routes_with_public_ip" { + type = bool + description = "(Optional) Whether subnet routes with public IP range are imported. " + default = null +} + +variable "stack_type" { + type = string + description = "(Optional) Which IP version(s) of traffic and routes are allowed to be imported or exported between peer networks. " + default = null +} + +resource "google_compute_network_peering" "peering" { + name = var.name + network = var.network_self_link + peer_network = var.peer_network_self_link + export_custom_routes = var.export_custom_routes + import_custom_routes = var.import_custom_routes + import_subnet_routes_with_public_ip = var.import_subnet_routes_with_public_ip + stack_type = var.stack_type +} + +output "peering_name" { + value = google_compute_network_peering.peering.name + description = "Name of the peering." +} + +terraform { + required_version = ">= 0.15.0" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/metadata.yaml new file mode 100644 index 0000000000..e80fc96b9c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/README.md b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/README.md new file mode 100644 index 0000000000..432442bf61 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/README.md @@ -0,0 +1,171 @@ +## Description + +This module simplifies the following functionality: + +* Applying Kubernetes manifests to GKE clusters: It provides flexible options for specifying manifests, allowing you to either directly embed them as strings content or reference them from URLs, files, templates, or entire .yaml and .tftpl files in directories. +* Deploying commonly used infrastructure like [Kueue](https://kueue.sigs.k8s.io/docs/) or [Jobset](https://jobset.sigs.k8s.io/docs/). + +> Note: Kueue can work with a variety of frameworks out of the box, find them [here](https://kueue.sigs.k8s.io/docs/tasks/run/) + +### Explanation + +* **Manifest:** + * **Raw String:** Specify manifests directly within the module configuration using the `content: manifest_body` format. + * **File/Template/Directory Reference:** Set `source` to the path to: + * A single URL to a manifest file. Ex.: `https://github.com/.../myrepo/manifest.yaml`. + * A single local YAML manifest file (`.yaml`). Ex.: `./manifest.yaml`. + * A template file (`.tftpl`) to generate a manifest. Ex.: `./template.yaml.tftpl`. You can pass the variables to format the template file in `template_vars`. + * A directory containing multiple YAML or template files. Ex: `./manifests/`. You can pass the variables to format the template files in `template_vars`. + +#### Manifest Example + +```yaml +- id: existing-gke-cluster + source: modules/scheduler/pre-existing-gke-cluster + settings: + project_id: $(vars.project_id) + cluster_name: my-gke-cluster + region: us-central1 + +- id: kubectl-apply + source: modules/management/kubectl-apply + use: [existing-gke-cluster] + settings: + - content: | + apiVersion: v1 + kind: Namespace + metadata: + name: my-namespace + - source: "https://github.com/kubernetes-sigs/jobset/releases/download/v0.6.0/manifests.yaml" + - source: $(ghpc_stage("manifests/configmap1.yaml")) + - source: $(ghpc_stage("manifests/configmap2.yaml.tftpl")) + template_vars: {name: "dev-config", public: "false"} + - source: $(ghpc_stage("manifests"))/ + template_vars: {name: "dev-config", public: "false"} +``` + +#### Pre-build infrastructure Example + +```yaml + - id: workload_component_install + source: modules/management/kubectl-apply + use: [gke_cluster] + settings: + kueue: + install: true + config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml")) + jobset: + install: true +``` + +The `config_path` field in `kueue` installation accepts a template file, too. You will need to provide variables for the template using `config_template_vars` field. + +```yaml + - id: workload_component_install + source: modules/management/kubectl-apply + use: [gke_cluster] + settings: + kueue: + install: true + config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml.tftpl")) + config_template_vars: {name: "dev-config", public: "false"} + jobset: + install: true +``` + +You can specify a particular kueue version that you would like to use using the `version` flag. By default, we recommend customers to [use v0.10.0](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/management/kubectl-apply/variables.tf#L68). You can find the list of supported kueue versions [here](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/management/kubectl-apply/variables.tf#L18). + +```yaml + - id: workload_component_install + source: modules/management/kubectl-apply + use: [gke_cluster] + settings: + kueue: + install: true + version: v0.10.0 + config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml.tftpl")) + config_template_vars: {name: "dev-config", public: "false"} + jobset: + install: true +``` + +> **_NOTE:_** +> +> The `project_id` and `region` settings would be inferred from the deployment variables of the same name, but they are included here for clarity. +> +> Terraform may apply resources in parallel, leading to potential dependency issues. If a resource's dependencies aren't ready, it will be applied again up to 15 times. + +## License + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | > 5.0 | +| [helm](#requirement\_helm) | ~> 2.17 | +| [kubectl](#requirement\_kubectl) | >= 1.7.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | > 5.0 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [configure\_kueue](#module\_configure\_kueue) | ./kubectl | n/a | +| [install\_gib](#module\_install\_gib) | ./kubectl | n/a | +| [install\_gpu\_operator](#module\_install\_gpu\_operator) | ./helm_install | n/a | +| [install\_jobset](#module\_install\_jobset) | ./kubectl | n/a | +| [install\_kueue](#module\_install\_kueue) | ./kubectl | n/a | +| [install\_nvidia\_dra\_driver](#module\_install\_nvidia\_dra\_driver) | ./helm_install | n/a | +| [kubectl\_apply\_manifests](#module\_kubectl\_apply\_manifests) | ./kubectl | n/a | + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.gib_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.initial_gib_version](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.jobset_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.kueue_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | +| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [apply\_manifests](#input\_apply\_manifests) | A list of manifests to apply to GKE cluster using kubectl. For more details see [kubectl module's inputs](kubectl/README.md). | 
list(object({
    enable            = optional(bool, true)
    content           = optional(string, null)
    source            = optional(string, null)
    template_vars     = optional(map(any), null)
    server_side_apply = optional(bool, false)
    wait_for_rollout  = optional(bool, true)
  }))
| `[]` | no | +| [cluster\_id](#input\_cluster\_id) | An identifier for the gke cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | +| [gib](#input\_gib) | Install the NCCL gIB plugin | 
object({
    install = bool
    path    = string
    template_vars = object({
      image   = optional(string, "us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib")
      version = string
      node_affinity = optional(any, {
        requiredDuringSchedulingIgnoredDuringExecution = {
          nodeSelectorTerms = [{
            matchExpressions = [{
              key      = "cloud.google.com/gke-gpu",
              operator = "In",
              values   = ["true"]
            }]
          }]
        }
      })
      accelerator_count = number
    })
  })
| 
{
  "install": false,
  "path": "",
  "template_vars": {
    "accelerator_count": 0,
    "version": ""
  }
}
| no | +| [gke\_cluster\_exists](#input\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | `bool` | `false` | no | +| [gpu\_operator](#input\_gpu\_operator) | Install [GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) which uses the [Kubernetes operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) to automate the management of all NVIDIA software components needed to provision GPU. | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | +| [jobset](#input\_jobset) | Install [Jobset](https://github.com/kubernetes-sigs/jobset) which manages a group of K8s [jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) as a unit. | 
object({
    install = optional(bool, false)
    version = optional(string, "v0.7.2")
  })
| `{}` | no | +| [kueue](#input\_kueue) | Install and configure [Kueue](https://kueue.sigs.k8s.io/docs/overview/) workload scheduler. A configuration yaml/template file can be provided with config\_path to be applied right after kueue installation. If a template file provided, its variables can be set to config\_template\_vars. | 
object({
    install              = optional(bool, false)
    version              = optional(string, "v0.12.2")
    config_path          = optional(string, null)
    config_template_vars = optional(map(any), null)
  })
| `{}` | no | +| [nvidia\_dra\_driver](#input\_nvidia\_dra\_driver) | Installs [Nvidia DRA driver](https://github.com/NVIDIA/k8s-dra-driver-gpu) which supports Dynamic Resource Allocation for NVIDIA GPUs in Kubernetes | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0-rc.3")
  })
| `{}` | no | +| [project\_id](#input\_project\_id) | The project ID that hosts the gke cluster. | `string` | n/a | yes | + +## Outputs + +No outputs. + diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/README.md b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/README.md new file mode 100644 index 0000000000..1957899617 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/README.md @@ -0,0 +1,64 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [helm](#requirement\_helm) | ~> 2.17 | + +## Providers + +| Name | Version | +|------|---------| +| [helm](#provider\_helm) | ~> 2.17 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [helm_release.apply_chart](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [atomic](#input\_atomic) | If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used. | `bool` | `false` | no | +| [chart\_name](#input\_chart\_name) | Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL). | `string` | n/a | yes | +| [chart\_repository](#input\_chart\_repository) | URL of the Helm chart repository. Set to null or omit if 'chart\_name' is a path or URL. | `string` | `null` | no | +| [chart\_version](#input\_chart\_version) | Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true). | `string` | `null` | no | +| [cleanup\_on\_fail](#input\_cleanup\_on\_fail) | Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail'). | `bool` | `false` | no | +| [create\_namespace](#input\_create\_namespace) | Set to true to create the namespace if it does not exist ('helm install --create-namespace'). | `bool` | `true` | no | +| [dependency\_update](#input\_dependency\_update) | Run 'helm dependency update' before installing the chart (useful if chart\_name is a local path to an unpacked chart with dependencies). | `bool` | `false` | no | +| [description](#input\_description) | Set an optional description for the Helm release. | `string` | `null` | no | +| [devel](#input\_devel) | Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart\_version' is set, this is ignored. | `bool` | `false` | no | +| [disable\_crd\_hooks](#input\_disable\_crd\_hooks) | Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook'). | `bool` | `false` | no | +| [disable\_openapi\_validation](#input\_disable\_openapi\_validation) | If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation'). | `bool` | `false` | no | +| [disable\_webhooks](#input\_disable\_webhooks) | Prevent hooks from running ('helm install --no-hooks'). | `bool` | `false` | no | +| [force\_update](#input\_force\_update) | Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution. | `bool` | `false` | no | +| [keyring](#input\_keyring) | Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true. | `string` | `null` | no | +| [lint](#input\_lint) | Run the helm chart linter during the plan ('helm lint'). | `bool` | `false` | no | +| [max\_history](#input\_max\_history) | Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit. | `number` | `null` | no | +| [namespace](#input\_namespace) | Kubernetes namespace to install the Helm release into. | `string` | `"default"` | no | +| [pass\_credentials](#input\_pass\_credentials) | Pass credentials to all domains ('helm install --pass-credentials'). Use with caution. | `bool` | `false` | no | +| [postrender](#input\_postrender) | Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary\_path' attribute. | 
object({
    binary_path = string # Path to the post-renderer executable
  })
| `null` | no | +| [recreate\_pods](#input\_recreate\_pods) | Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself. | `bool` | `false` | no | +| [release\_name](#input\_release\_name) | Name of the Helm release. | `string` | n/a | yes | +| [render\_subchart\_notes](#input\_render\_subchart\_notes) | If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes'). | `bool` | `false` | no | +| [reset\_values](#input\_reset\_values) | When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values'). | `bool` | `false` | no | +| [reuse\_values](#input\_reuse\_values) | When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset\_values' is specified, this is ignored. | `bool` | `false` | no | +| [set\_values](#input\_set\_values) | List of objects defining values to set ('helm install --set'). | 
list(object({
    name  = string                     # Path to the value (e.g., 'service.type', 'replicaCount')
    value = string                     # The value to set
    type  = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file')
  }))
| `[]` | no | +| [skip\_crds](#input\_skip\_crds) | If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present. | `bool` | `false` | no | +| [timeout](#input\_timeout) | Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout'). | `number` | `300` | no | +| [values\_yaml](#input\_values\_yaml) | List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile(). | `list(string)` | `[]` | no | +| [verify](#input\_verify) | Verify the package before installing it ('helm install --verify'). | `bool` | `false` | no | +| [wait](#input\_wait) | Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait'). | `bool` | `true` | no | +| [wait\_for\_jobs](#input\_wait\_for\_jobs) | If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs'). | `bool` | `false` | no | + +## Outputs + +No outputs. + diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf new file mode 100644 index 0000000000..8cc09bd3e2 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf @@ -0,0 +1,79 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "helm_release" "apply_chart" { + # Required Identification + name = var.release_name + chart = var.chart_name + + # Chart Source & Version + repository = var.chart_repository + version = var.chart_version + devel = var.devel + + # Target Namespace + namespace = var.namespace + create_namespace = var.create_namespace + + # Values Configuration + values = var.values_yaml + + dynamic "set" { + for_each = var.set_values + content { + name = set.value.name + value = set.value.value + type = set.value.type + } + } + + # Installation/Upgrade Behavior + description = var.description + atomic = var.atomic + cleanup_on_fail = var.cleanup_on_fail + dependency_update = var.dependency_update + disable_crd_hooks = var.disable_crd_hooks + disable_openapi_validation = var.disable_openapi_validation + disable_webhooks = var.disable_webhooks + force_update = var.force_update + lint = var.lint + max_history = var.max_history + recreate_pods = var.recreate_pods # Note: Deprecated in Helm CLI + render_subchart_notes = var.render_subchart_notes + reset_values = var.reset_values + reuse_values = var.reuse_values + skip_crds = var.skip_crds + timeout = var.timeout + wait = var.wait + wait_for_jobs = var.wait_for_jobs + + # Verification & Credentials + keyring = var.keyring + pass_credentials = var.pass_credentials + verify = var.verify + + # Post Rendering + dynamic "postrender" { + # Only include the block if var.postrender is not null + for_each = var.postrender == null ? [] : [var.postrender] + content { + binary_path = postrender.value.binary_path + } + } + + # Lifecycle block (optional - generally avoid complex lifecycle in generic modules) + # lifecycle { + # ignore_changes = [] + # } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml new file mode 100644 index 0000000000..17bedb471b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf new file mode 100644 index 0000000000..04e8e214fc --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf @@ -0,0 +1,212 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Description: Input variables for the generic Helm release module. + +# --- Required --- +variable "release_name" { + description = "Name of the Helm release." + type = string +} + +variable "chart_name" { + description = "Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL)." + type = string +} + +# --- Chart Location & Version --- +variable "chart_repository" { + description = "URL of the Helm chart repository. Set to null or omit if 'chart_name' is a path or URL." + type = string + default = null +} + +variable "chart_version" { + description = "Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true)." + type = string + default = null +} + +variable "devel" { + description = "Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart_version' is set, this is ignored." + type = bool + default = false +} + +# --- Namespace --- +variable "namespace" { + description = "Kubernetes namespace to install the Helm release into." + type = string + default = "default" +} + +variable "create_namespace" { + description = "Set to true to create the namespace if it does not exist ('helm install --create-namespace')." + type = bool + default = true # Common convenience setting +} + +# --- Values Customization --- +variable "values_yaml" { + description = "List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile()." + type = list(string) + default = [] +} + +variable "set_values" { + description = "List of objects defining values to set ('helm install --set')." + type = list(object({ + name = string # Path to the value (e.g., 'service.type', 'replicaCount') + value = string # The value to set + type = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file') + })) + default = [] +} + +# --- Installation/Upgrade Behavior --- +variable "description" { + description = "Set an optional description for the Helm release." + type = string + default = null +} + +variable "atomic" { + description = "If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used." + type = bool + default = false +} + +variable "wait" { + description = "Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait')." + type = bool + default = true # Often a good default for dependencies +} + +variable "wait_for_jobs" { + description = "If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs')." + type = bool + default = false # Helm CLI default is false +} + +variable "timeout" { + description = "Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout')." + type = number + default = 300 # 5 minutes (Helm CLI default) +} + +variable "cleanup_on_fail" { + description = "Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail')." + type = bool + default = false +} + +variable "dependency_update" { + description = "Run 'helm dependency update' before installing the chart (useful if chart_name is a local path to an unpacked chart with dependencies)." + type = bool + default = false +} + +variable "disable_crd_hooks" { + description = "Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook')." + type = bool + default = false +} + +variable "disable_openapi_validation" { + description = "If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation')." + type = bool + default = false +} + +variable "disable_webhooks" { + description = "Prevent hooks from running ('helm install --no-hooks')." + type = bool + default = false +} + +variable "force_update" { + description = "Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution." + type = bool + default = false +} + +variable "lint" { + description = "Run the helm chart linter during the plan ('helm lint')." + type = bool + default = false +} + +variable "max_history" { + description = "Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit." + type = number + default = null # Terraform provider defaults to Helm's default (usually 10) +} + +variable "recreate_pods" { + description = "Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself." + type = bool + default = false +} + +variable "render_subchart_notes" { + description = "If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes')." + type = bool + default = false +} + +variable "reset_values" { + description = "When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values')." + type = bool + default = false +} + +variable "reuse_values" { + description = "When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset_values' is specified, this is ignored." + type = bool + default = false # Helm CLI default is false +} + +variable "skip_crds" { + description = "If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present." + type = bool + default = false +} + +# --- Verification & Credentials --- +variable "keyring" { + description = "Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true." + type = string + default = null # Defaults to Helm's default keyring location +} + +variable "pass_credentials" { + description = "Pass credentials to all domains ('helm install --pass-credentials'). Use with caution." + type = bool + default = false +} + +variable "verify" { + description = "Verify the package before installing it ('helm install --verify')." + type = bool + default = false +} + +# --- Advanced Rendering --- +variable "postrender" { + description = "Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary_path' attribute." + type = object({ + binary_path = string # Path to the post-renderer executable + }) + default = null # Disabled by default +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf new file mode 100644 index 0000000000..09d912e2c9 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf @@ -0,0 +1,24 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + helm = { + source = "hashicorp/helm" + version = "~> 2.17" + } + } + + required_version = ">= 1.3" +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/README.md b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/README.md new file mode 100644 index 0000000000..a57421b81f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/README.md @@ -0,0 +1,57 @@ + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [http](#requirement\_http) | ~> 3.0 | +| [kubectl](#requirement\_kubectl) | >= 1.7.0 | + +## Providers + +| Name | Version | +|------|---------| +| [http](#provider\_http) | ~> 3.0 | +| [kubectl](#provider\_kubectl) | >= 1.7.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [kubectl_manifest.apply_doc](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [http_http.yaml_content](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | +| [kubectl_path_documents.templates](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/path_documents) | data source | +| [kubectl_path_documents.yamls](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/path_documents) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [content](#input\_content) | The YAML body to apply to gke cluster. | `string` | `null` | no | +| [server\_side\_apply](#input\_server\_side\_apply) | Allow using kubectl server-side apply method. | `bool` | `false` | no | +| [source\_path](#input\_source\_path) | The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file. | `string` | `null` | no | +| [template\_vars](#input\_template\_vars) | The values to populate template file(s) with. | `any` | `null` | no | +| [wait\_for\_rollout](#input\_wait\_for\_rollout) | Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details. | `bool` | `true` | no | + +## Outputs + +No outputs. + diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf new file mode 100644 index 0000000000..c2abe3ae94 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + yaml_separator = "\n---" + + content_yaml_body = var.content + + null_safe_source = coalesce(var.source_path, " ") + + url = startswith(local.null_safe_source, "http://") || startswith(local.null_safe_source, "https://") ? var.source_path : null + url_content = local.url != null ? data.http.yaml_content[0].response_body : null + + yaml_file = local.url == null && length(regexall("\\.yaml(_.*)?$", lower(local.null_safe_source))) == 1 ? abspath(var.source_path) : null + yaml_file_content = local.yaml_file != null ? file(local.yaml_file) : null + + template_file = local.url == null && length(regexall("\\.tftpl(_.*)?$", lower(local.null_safe_source))) == 1 ? abspath(var.source_path) : null + template_file_content = local.template_file != null ? templatefile(local.template_file, var.template_vars) : null + + yaml_body = coalesce(local.content_yaml_body, local.url_content, local.yaml_file_content, local.template_file_content, " ") + yaml_body_docs = [for doc in split(local.yaml_separator, local.yaml_body) : trimspace(doc) if length(trimspace(doc)) > 0] # Split yaml to single docs because the kubectl provider only supports single resource + + directory = length(local.yaml_body_docs) == 0 && endswith(local.null_safe_source, "/") ? abspath(var.source_path) : null + + docs_list = concat(try(local.yaml_body_docs, []), try(data.kubectl_path_documents.yamls[0].documents, []), try(data.kubectl_path_documents.templates[0].documents, [])) + docs_map = tomap({ + for index, doc in local.docs_list : index => doc + }) +} + +data "http" "yaml_content" { + count = local.url != null ? 1 : 0 + url = local.url +} + +data "kubectl_path_documents" "yamls" { + count = local.directory != null ? 1 : 0 + pattern = "${local.directory}/*.yaml" +} + +data "kubectl_path_documents" "templates" { + count = local.directory != null ? 1 : 0 + pattern = "${local.directory}/*.tftpl" + vars = var.template_vars +} + +resource "kubectl_manifest" "apply_doc" { + for_each = local.docs_map + yaml_body = each.value + server_side_apply = var.server_side_apply + wait_for_rollout = var.wait_for_rollout +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml new file mode 100644 index 0000000000..17bedb471b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf new file mode 100644 index 0000000000..54a996b8f3 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf @@ -0,0 +1,45 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "content" { + description = "The YAML body to apply to gke cluster." + type = string + default = null +} + +variable "source_path" { + description = "The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file." + type = string + default = null +} + +variable "template_vars" { + description = "The values to populate template file(s) with." + type = any + default = null +} + +variable "server_side_apply" { + description = "Allow using kubectl server-side apply method." + type = bool + default = false +} + +variable "wait_for_rollout" { + description = "Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details." + type = bool + default = true +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf new file mode 100644 index 0000000000..0275fb90a7 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + kubectl = { + source = "gavinbunney/kubectl" + version = ">= 1.7.0" + } + http = { + source = "hashicorp/http" + version = "~> 3.0" + } + } + + required_version = ">= 1.3" +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/main.tf b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/main.tf new file mode 100644 index 0000000000..db85c65f32 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/main.tf @@ -0,0 +1,234 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_id_parts = split("/", var.cluster_id) + cluster_name = local.cluster_id_parts[5] + cluster_location = local.cluster_id_parts[3] + project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] + + apply_manifests_map = tomap({ + for index, manifest in var.apply_manifests : index => manifest + if manifest.enable + }) + + install_kueue = try(var.kueue.install, false) + install_jobset = try(var.jobset.install, false) + install_gpu_operator = try(var.gpu_operator.install, false) + install_nvidia_dra_driver = try(var.nvidia_dra_driver.install, false) + install_gib = try(var.gib.install, false) + kueue_install_source = format("${path.module}/manifests/kueue-%s.yaml", try(var.kueue.version, "")) + jobset_install_source = format("${path.module}/manifests/jobset-%s.yaml", try(var.jobset.version, "")) +} + +data "google_container_cluster" "gke_cluster" { + project = local.project_id + name = local.cluster_name + location = local.cluster_location +} + +data "google_client_config" "default" {} + +module "kubectl_apply_manifests" { + for_each = local.apply_manifests_map + source = "./kubectl" + depends_on = [var.gke_cluster_exists] + + content = each.value.content + source_path = each.value.source + template_vars = each.value.template_vars + server_side_apply = each.value.server_side_apply + wait_for_rollout = each.value.wait_for_rollout + + providers = { + kubectl = kubectl + } +} + +module "install_kueue" { + source = "./kubectl" + source_path = local.install_kueue ? local.kueue_install_source : null + server_side_apply = true + wait_for_rollout = true + depends_on = [var.gke_cluster_exists] + + providers = { + kubectl = kubectl + } +} + +module "configure_kueue" { + source = "./kubectl" + source_path = local.install_kueue ? try(var.kueue.config_path, "") : null + template_vars = local.install_kueue ? try(var.kueue.config_template_vars, null) : null + depends_on = [module.install_kueue] + + server_side_apply = true + wait_for_rollout = true + + providers = { + kubectl = kubectl + } +} + +module "install_jobset" { + source = "./kubectl" + source_path = local.install_jobset ? local.jobset_install_source : null + server_side_apply = true + wait_for_rollout = true + depends_on = [var.gke_cluster_exists, module.configure_kueue] + + providers = { + kubectl = kubectl + } +} + +module "install_nvidia_dra_driver" { + count = local.install_nvidia_dra_driver ? 1 : 0 + depends_on = [module.kubectl_apply_manifests, var.gke_cluster_exists, module.configure_kueue] + source = "./helm_install" + + release_name = "nvidia-dra-driver-gpu" # The release name + chart_repository = "https://helm.ngc.nvidia.com/nvidia" # The Helm repository URL for nvidia charts + chart_name = "nvidia-dra-driver-gpu" # The chart name + chart_version = var.nvidia_dra_driver.version # The chart version + namespace = "nvidia-dra-driver-gpu" # The target namespace + create_namespace = true # Equivalent to --create-namespace + + # Use the 'values' argument to pass the YAML content + # This corresponds to the -f <(cat <---. + type: boolean + subdomain: + description: |- + Subdomain is an explicit choice for a network subdomain name + When set, any replicated job in the set is added to this network. + Defaults to if not set. + type: string + type: object + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + replicatedJobs: + description: ReplicatedJobs is the group of jobs that will form the set. + items: + properties: + name: + description: |- + Name is the name of the entry and will be used as a suffix + for the Job name. + type: string + replicas: + default: 1 + description: |- + Replicas is the number of jobs that will be created from this ReplicatedJob's template. + Jobs names will be in the format: -- + format: int32 + type: integer + template: + description: Template defines the template of the Job that will be created. + properties: + metadata: + description: |- + Standard object's metadata of the jobs created from this template. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + Specification of the desired behavior of the job. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Specifies the duration in seconds relative to the startTime that the job + may be continuously active before the system tries to terminate it; value + must be positive integer. If a Job is suspended (at creation or through an + update), this timer will effectively be stopped and reset when the Job is + resumed again. + format: int64 + type: integer + backoffLimit: + description: |- + Specifies the number of retries before marking this job failed. + Defaults to 6 + format: int32 + type: integer + backoffLimitPerIndex: + description: |- + Specifies the limit for the number of retries within an + index before marking this index as failed. When enabled the number of + failures per index is kept in the pod's + batch.kubernetes.io/job-index-failure-count annotation. It can only + be set when Job's completionMode=Indexed, and the Pod's restart + policy is Never. The field is immutable. + This field is beta-level. It can be used when the `JobBackoffLimitPerIndex` + feature gate is enabled (enabled by default). + format: int32 + type: integer + completionMode: + description: |- + completionMode specifies how Pod completions are tracked. It can be + `NonIndexed` (default) or `Indexed`. + + + `NonIndexed` means that the Job is considered complete when there have + been .spec.completions successfully completed Pods. Each Pod completion is + homologous to each other. + + + `Indexed` means that the Pods of a + Job get an associated completion index from 0 to (.spec.completions - 1), + available in the annotation batch.kubernetes.io/job-completion-index. + The Job is considered complete when there is one successfully completed Pod + for each index. + When value is `Indexed`, .spec.completions must be specified and + `.spec.parallelism` must be less than or equal to 10^5. + In addition, The Pod name takes the form + `$(job-name)-$(index)-$(random-string)`, + the Pod hostname takes the form `$(job-name)-$(index)`. + + + More completion modes can be added in the future. + If the Job controller observes a mode that it doesn't recognize, which + is possible during upgrades due to version skew, the controller + skips updates for the Job. + type: string + completions: + description: |- + Specifies the desired number of successfully finished pods the + job should be run with. Setting to null means that the success of any + pod signals the success of all pods, and allows parallelism to have any positive + value. Setting to 1 means that parallelism is limited to 1 and the success of that + pod signals the success of the job. + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ + format: int32 + type: integer + manualSelector: + description: |- + manualSelector controls generation of pod labels and pod selectors. + Leave `manualSelector` unset unless you are certain what you are doing. + When false or unset, the system pick labels unique to this job + and appends those labels to the pod template. When true, + the user is responsible for picking unique labels and specifying + the selector. Failure to pick a unique label may cause this + and other jobs to not function correctly. However, You may see + `manualSelector=true` in jobs that were created with the old `extensions/v1beta1` + API. + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#specifying-your-own-pod-selector + type: boolean + maxFailedIndexes: + description: |- + Specifies the maximal number of failed indexes before marking the Job as + failed, when backoffLimitPerIndex is set. Once the number of failed + indexes exceeds this number the entire Job is marked as Failed and its + execution is terminated. When left as null the job continues execution of + all of its indexes and is marked with the `Complete` Job condition. + It can only be specified when backoffLimitPerIndex is set. + It can be null or up to completions. It is required and must be + less than or equal to 10^4 when is completions greater than 10^5. + This field is beta-level. It can be used when the `JobBackoffLimitPerIndex` + feature gate is enabled (enabled by default). + format: int32 + type: integer + parallelism: + description: |- + Specifies the maximum desired number of pods the job should + run at any given time. The actual number of pods running in steady state will + be less than this number when ((.spec.completions - .status.successful) < .spec.parallelism), + i.e. when the work left to do is less than max parallelism. + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ + format: int32 + type: integer + podFailurePolicy: + description: |- + Specifies the policy of handling failed pods. In particular, it allows to + specify the set of actions and conditions which need to be + satisfied to take the associated action. + If empty, the default behaviour applies - the counter of failed pods, + represented by the jobs's .status.failed field, is incremented and it is + checked against the backoffLimit. This field cannot be used in combination + with restartPolicy=OnFailure. + + + This field is beta-level. It can be used when the `JobPodFailurePolicy` + feature gate is enabled (enabled by default). + properties: + rules: + description: |- + A list of pod failure policy rules. The rules are evaluated in order. + Once a rule matches a Pod failure, the remaining of the rules are ignored. + When no rule matches the Pod failure, the default handling applies - the + counter of pod failures is incremented and it is checked against + the backoffLimit. At most 20 elements are allowed. + items: + description: |- + PodFailurePolicyRule describes how a pod failure is handled when the requirements are met. + One of onExitCodes and onPodConditions, but not both, can be used in each rule. + properties: + action: + description: |- + Specifies the action taken on a pod failure when the requirements are satisfied. + Possible values are: + + + - FailJob: indicates that the pod's job is marked as Failed and all + running pods are terminated. + - FailIndex: indicates that the pod's index is marked as Failed and will + not be restarted. + This value is beta-level. It can be used when the + `JobBackoffLimitPerIndex` feature gate is enabled (enabled by default). + - Ignore: indicates that the counter towards the .backoffLimit is not + incremented and a replacement pod is created. + - Count: indicates that the pod is handled in the default way - the + counter towards the .backoffLimit is incremented. + Additional values are considered to be added in the future. Clients should + react to an unknown action by skipping the rule. + type: string + onExitCodes: + description: Represents the requirement on the container exit codes. + properties: + containerName: + description: |- + Restricts the check for exit codes to the container with the + specified name. When null, the rule applies to all containers. + When specified, it should match one the container or initContainer + names in the pod template. + type: string + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Containers completed with success (exit code 0) are + excluded from the requirement check. Possible values are: + + + - In: the requirement is satisfied if at least one container exit code + (might be multiple if there are multiple containers not restricted + by the 'containerName' field) is in the set of specified values. + - NotIn: the requirement is satisfied if at least one container exit code + (might be multiple if there are multiple containers not restricted + by the 'containerName' field) is not in the set of specified values. + Additional values are considered to be added in the future. Clients should + react to an unknown operator by assuming the requirement is not satisfied. + type: string + values: + description: |- + Specifies the set of values. Each returned container exit code (might be + multiple in case of multiple containers) is checked against this set of + values with respect to the operator. The list of values must be ordered + and must not contain duplicates. Value '0' cannot be used for the In operator. + At least one element is required. At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + - values + type: object + onPodConditions: + description: |- + Represents the requirement on the pod conditions. The requirement is represented + as a list of pod condition patterns. The requirement is satisfied if at + least one pattern matches an actual pod condition. At most 20 elements are allowed. + items: + description: |- + PodFailurePolicyOnPodConditionsPattern describes a pattern for matching + an actual pod condition type. + properties: + status: + description: |- + Specifies the required Pod condition status. To match a pod condition + it is required that the specified status equals the pod condition status. + Defaults to True. + type: string + type: + description: |- + Specifies the required Pod condition type. To match a pod condition + it is required that specified type equals the pod condition type. + type: string + required: + - status + - type + type: object + type: array + x-kubernetes-list-type: atomic + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + required: + - rules + type: object + podReplacementPolicy: + description: |- + podReplacementPolicy specifies when to create replacement Pods. + Possible values are: + - TerminatingOrFailed means that we recreate pods + when they are terminating (has a metadata.deletionTimestamp) or failed. + - Failed means to wait until a previously created Pod is fully terminated (has phase + Failed or Succeeded) before creating a replacement Pod. + + + When using podFailurePolicy, Failed is the the only allowed value. + TerminatingOrFailed and Failed are allowed values when podFailurePolicy is not in use. + This is an beta field. To use this, enable the JobPodReplacementPolicy feature toggle. + This is on by default. + type: string + selector: + description: |- + A label query over pods that should match the pod count. + Normally, the system sets this field for you. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + suspend: + description: |- + suspend specifies whether the Job controller should create Pods or not. If + a Job is created with suspend set to true, no Pods are created by the Job + controller. If a Job is suspended after creation (i.e. the flag goes from + false to true), the Job controller will delete all active Pods associated + with this Job. Users must design their workload to gracefully handle this. + Suspending a Job will reset the StartTime field of the Job, effectively + resetting the ActiveDeadlineSeconds timer too. Defaults to false. + type: boolean + template: + description: |- + Describes the pod that will be created when executing a job. + The only allowed template.spec.restartPolicy values are "Never" or "OnFailure". + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates whether a service account token should be automatically mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container that you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present in a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration that the container should sleep before being terminated. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration that the container should sleep before being terminated. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default is DefaultProcMount which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. + items: + description: volumeDevice describes a mapping of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS resolver options of a pod. + properties: + name: + description: Required. + type: string + value: + type: string + type: object + type: array + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present in a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed for ephemeral containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration that the container should sleep before being terminated. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration that the container should sleep before being terminated. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral containers. + items: + description: ContainerPort represents a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + This may only be set for init containers. You cannot set this field on + ephemeral containers. + type: string + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default is DefaultProcMount which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. + items: + description: volumeDevice describes a mapping of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. This is only valid for non-hostNetwork pods. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + ip: + description: IP address of the host file entry. + type: string + type: object + type: array + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + If this option is set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + type: array + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + of that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container that you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present in a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration that the container should sleep before being terminated. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration that the container should sleep before being terminated. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default is DefaultProcMount which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. + items: + description: volumeDevice describes a mapping of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + nodeName: + description: |- + NodeName is a request to schedule this pod onto a specific node. If it is non-empty, + the scheduler simply schedules this pod onto that node, assuming that it fits resource + requirements. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the reference to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition in the pod's condition list with matching type. + type: string + required: + - conditionType + type: object + type: array + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim through a ClaimSource. + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + source: + description: Source describes where to find the ResourceClaim. + properties: + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + type: string + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + + + This is a beta feature enabled by the PodSchedulingReadiness feature gate. + items: + description: PodSchedulingGate is associated to a Pod to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in addition + to the container's primary GID, the fsGroup (if specified), and group memberships + defined in the container image for the uid of the container process. If unspecified, + no additional groups are added to any container. Note that group memberships + defined in the container image for the uid of the container process are still effective, + even if they are not included in this list. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a depreciated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + + + This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume in a pod that may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem from compromising the machine + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching mode: None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk in the blob storage + type: string + fsType: + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared' + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File Service mount on the host and bind mount to the pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret that contains Azure Storage Account Name and Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount on the host that shares a pod's lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + path: + description: 'path is Optional: Used as the mounted root, rather than the full Ceph tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: optional specify whether the ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents ephemeral storage that is handled by certain external CSI drivers (Beta feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API about the pod that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API volume file + items: + description: DownwardAPIVolumeFile represents information to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#volumeattributesclass + (Alpha) Using this field requires the VolumeAttributesClass feature gate to be enabled. + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + TODO: how do we prevent errors in the filesystem from compromising the machine + type: string + lun: + description: 'lun is Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target worldwide names (WWNs)' + items: + type: string + type: array + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field holds extra command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the dataset. This is unique identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem from compromising the machine + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- + TODO(jonesdl) We need to restrict who can use host directory mounts and who can/can not + mount host directories as read/write. + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem from compromising the machine + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified Name. + type: string + iscsiInterface: + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for iSCSI target and initiator authentication + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx volume attached and mounted on kubelets host machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources secrets, configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: sources is the list of volume projections + items: + description: Projection that may be projected along with other supported volume types + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the volume root to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: optional specify whether the ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about the downwardAPI data to project + properties: + items: + description: Items is a list of DownwardAPIVolume file + items: + description: DownwardAPIVolumeFile represents information to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + secret: + description: secret information about the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: optional field specify whether the Secret or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information about the serviceAccountToken data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + description: quobyte represents a Quobyte mount on the host that shares a pod's lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem from compromising the machine + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + pool: + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of the ScaleIO Protection Domain for the configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable SSL communication with Gateway, default false + type: boolean + storageMode: + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage Pool associated with the protection domain. + type: string + system: + description: system is the name of the storage system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + optional: + description: optional field specify whether the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage Policy Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + required: + - containers + type: object + type: object + ttlSecondsAfterFinished: + description: |- + ttlSecondsAfterFinished limits the lifetime of a Job that has finished + execution (either Complete or Failed). If this field is set, + ttlSecondsAfterFinished after the Job finishes, it is eligible to be + automatically deleted. When the Job is being deleted, its lifecycle + guarantees (e.g. finalizers) will be honored. If this field is unset, + the Job won't be automatically deleted. If this field is set to zero, + the Job becomes eligible to be deleted immediately after it finishes. + format: int32 + type: integer + required: + - template + type: object + type: object + required: + - name + - template + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + startupPolicy: + description: StartupPolicy, if set, configures in what order jobs must be started + properties: + startupPolicyOrder: + description: |- + StartupPolicyOrder determines the startup order of the ReplicatedJobs. + AnyOrder means to start replicated jobs in any order. + InOrder means to start them as they are listed in the JobSet. A ReplicatedJob is started only + when all the jobs of the previous one are ready. + enum: + - AnyOrder + - InOrder + type: string + required: + - startupPolicyOrder + type: object + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + successPolicy: + description: |- + SuccessPolicy configures when to declare the JobSet as + succeeded. + The JobSet is always declared succeeded if all jobs in the set + finished with status complete. + properties: + operator: + description: Operator determines either All or Any of the selected jobs should succeed to consider the JobSet successful + enum: + - All + - Any + type: string + targetReplicatedJobs: + description: |- + TargetReplicatedJobs are the names of the replicated jobs the operator will apply to. + A null or empty list will apply to all replicatedJobs. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - operator + type: object + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + suspend: + description: Suspend suspends all running child Jobs when set to true. + type: boolean + ttlSecondsAfterFinished: + description: |- + TTLSecondsAfterFinished limits the lifetime of a JobSet that has finished + execution (either Complete or Failed). If this field is set, + TTLSecondsAfterFinished after the JobSet finishes, it is eligible to be + automatically deleted. When the JobSet is being deleted, its lifecycle + guarantees (e.g. finalizers) will be honored. If this field is unset, + the JobSet won't be automatically deleted. If this field is set to zero, + the JobSet becomes eligible to be deleted immediately after it finishes. + format: int32 + minimum: 0 + type: integer + type: object + status: + description: JobSetStatus defines the observed state of JobSet + properties: + conditions: + items: + description: "Condition contains details for one aspect of the current state of this API Resource.\n---\nThis struct is intended for direct use as an array at the field path .status.conditions. For example,\n\n\n\ttype FooStatus struct{\n\t // Represents the observations of a foo's current state.\n\t // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + replicatedJobsStatus: + description: ReplicatedJobsStatus track the number of JobsReady for each replicatedJob. + items: + description: ReplicatedJobStatus defines the observed ReplicatedJobs Readiness. + properties: + active: + description: |- + Active is the number of child Jobs with at least 1 pod in a running or pending state + which are not marked for deletion. + format: int32 + type: integer + failed: + description: Failed is the number of failed child Jobs. + format: int32 + type: integer + name: + description: Name of the ReplicatedJob. + type: string + ready: + description: |- + Ready is the number of child Jobs where the number of ready pods and completed pods + is greater than or equal to the total expected pod count for the Job (i.e., the minimum + of job.spec.parallelism and job.spec.completions). + format: int32 + type: integer + succeeded: + description: Succeeded is the number of successfully completed child Jobs. + format: int32 + type: integer + suspended: + description: Suspended is the number of child Jobs which are in a suspended state. + format: int32 + type: integer + required: + - active + - failed + - name + - ready + - succeeded + - suspended + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + restarts: + description: Restarts tracks the number of times the JobSet has restarted (i.e. recreated in case of RecreateAll policy). + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: controller-manager + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: serviceaccount + app.kubernetes.io/part-of: jobset + name: jobset-controller-manager + namespace: jobset-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: leader-election-role + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: role + app.kubernetes.io/part-of: jobset + name: jobset-leader-election-role + namespace: jobset-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: jobset-manager-role +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + verbs: + - get + - list + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - get + - list + - update + - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch + resources: + - jobs/status + verbs: + - get + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/finalizers + verbs: + - update +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/status + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: metrics-reader + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clusterrole + app.kubernetes.io/part-of: jobset + name: jobset-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: proxy-role + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clusterrole + app.kubernetes.io/part-of: jobset + name: jobset-proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: leader-election-rolebinding + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: rolebinding + app.kubernetes.io/part-of: jobset + name: jobset-leader-election-rolebinding + namespace: jobset-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: jobset-leader-election-role +subjects: +- kind: ServiceAccount + name: jobset-controller-manager + namespace: jobset-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: manager-rolebinding + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clusterrolebinding + app.kubernetes.io/part-of: jobset + name: jobset-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: jobset-manager-role +subjects: +- kind: ServiceAccount + name: jobset-controller-manager + namespace: jobset-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: proxy-rolebinding + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clusterrolebinding + app.kubernetes.io/part-of: jobset + name: jobset-proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: jobset-proxy-role +subjects: +- kind: ServiceAccount + name: jobset-controller-manager + namespace: jobset-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: jobset-webhook-server-cert + namespace: jobset-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: controller-manager-metrics-service + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: service + app.kubernetes.io/part-of: jobset + control-plane: controller-manager + name: jobset-controller-manager-metrics-service + namespace: jobset-system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + control-plane: controller-manager +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: webhook-service + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: service + app.kubernetes.io/part-of: jobset + name: jobset-webhook-service + namespace: jobset-system +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: manager + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: controller-manager + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: deployment + app.kubernetes.io/part-of: jobset + control-plane: controller-manager + name: jobset-controller-manager + namespace: jobset-system +spec: + replicas: 1 + selector: + matchLabels: + control-plane: controller-manager + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + control-plane: controller-manager + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - arm64 + - ppc64le + - s390x + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - args: + - --zap-log-level=2 + - --health-probe-bind-address=:8081 + - --metrics-bind-address=127.0.0.1:8080 + - --leader-elect + command: + - /manager + image: registry.k8s.io/jobset/jobset:v0.5.2 + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 2 + memory: 512Mi + requests: + cpu: 500m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + - args: + - --secure-listen-address=0.0.0.0:8443 + - --upstream=http://127.0.0.1:8080/ + - --logtostderr=true + - --v=0 + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1 + name: kube-rbac-proxy + ports: + - containerPort: 8443 + name: https + protocol: TCP + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + securityContext: + runAsNonRoot: true + serviceAccountName: jobset-controller-manager + terminationGracePeriodSeconds: 10 + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: jobset-webhook-server-cert + tolerations: + - effect: NoSchedule + key: components.gke.io/gke-managed-components + operator: Equal + value: "true" +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: jobset-mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: jobset-webhook-service + namespace: jobset-system + path: /mutate-jobset-x-k8s-io-v1alpha2-jobset + failurePolicy: Fail + name: mjobset.kb.io + rules: + - apiGroups: + - jobset.x-k8s.io + apiVersions: + - v1alpha2 + operations: + - CREATE + - UPDATE + resources: + - jobsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: jobset-webhook-service + namespace: jobset-system + path: /mutate--v1-pod + failurePolicy: Fail + name: mpod.kb.io + objectSelector: + matchExpressions: + - key: jobset.sigs.k8s.io/jobset-name + operator: Exists + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: jobset-validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: jobset-webhook-service + namespace: jobset-system + path: /validate-jobset-x-k8s-io-v1alpha2-jobset + failurePolicy: Fail + name: vjobset.kb.io + rules: + - apiGroups: + - jobset.x-k8s.io + apiVersions: + - v1alpha2 + operations: + - CREATE + - UPDATE + resources: + - jobsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: jobset-webhook-service + namespace: jobset-system + path: /validate--v1-pod + failurePolicy: Fail + name: vpod.kb.io + objectSelector: + matchExpressions: + - key: jobset.sigs.k8s.io/jobset-name + operator: Exists + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/jobset-v0.7.2.yaml b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/jobset-v0.7.2.yaml new file mode 100644 index 0000000000..a600eaf308 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/jobset-v0.7.2.yaml @@ -0,0 +1,8918 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + labels: + app.kubernetes.io/component: manager + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: system + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: namespace + app.kubernetes.io/part-of: jobset + control-plane: controller-manager + name: jobset-system +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + controller-gen.kubebuilder.io/version: v0.16.2 + name: jobsets.jobset.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: jobset-webhook-service + namespace: jobset-system + path: /convert + conversionReviewVersions: + - v1 + group: jobset.x-k8s.io + names: + kind: JobSet + listKind: JobSetList + plural: jobsets + singular: jobset + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Final state of JobSet + jsonPath: .status.terminalState + name: TerminalState + type: string + - description: Number of restarts + jsonPath: .status.restarts + name: Restarts + type: string + - jsonPath: .status.conditions[?(@.type=="Completed")].status + name: Completed + type: string + - description: JobSet suspended + jsonPath: .spec.suspend + name: Suspended + type: string + - description: Time this JobSet was created + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha2 + schema: + openAPIV3Schema: + description: JobSet is the Schema for the jobsets API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: JobSetSpec defines the desired state of JobSet + properties: + coordinator: + description: |- + Coordinator can be used to assign a specific pod as the coordinator for + the JobSet. If defined, an annotation will be added to all Jobs and pods with + coordinator pod, which contains the stable network endpoint where the + coordinator pod can be reached. + jobset.sigs.k8s.io/coordinator=. + properties: + jobIndex: + description: |- + JobIndex is the index of Job which contains the coordinator pod + (i.e., for a ReplicatedJob with N replicas, there are Job indexes 0 to N-1). + type: integer + podIndex: + description: PodIndex is the Job completion index of the coordinator pod. + type: integer + replicatedJob: + description: |- + ReplicatedJob is the name of the ReplicatedJob which contains + the coordinator pod. + type: string + required: + - replicatedJob + type: object + failurePolicy: + description: |- + FailurePolicy, if set, configures when to declare the JobSet as + failed. + The JobSet is always declared failed if any job in the set + finished with status failed. + properties: + maxRestarts: + description: |- + MaxRestarts defines the limit on the number of JobSet restarts. + A restart is achieved by recreating all active child jobs. + format: int32 + type: integer + restartStrategy: + default: Recreate + description: |- + RestartStrategy defines the strategy to use when restarting the JobSet. + Defaults to Recreate. + enum: + - Recreate + - BlockingRecreate + type: string + rules: + description: |- + List of failure policy rules for this JobSet. + For a given Job failure, the rules will be evaluated in order, + and only the first matching rule will be executed. + If no matching rule is found, the RestartJobSet action is applied. + items: + description: |- + FailurePolicyRule defines a FailurePolicyAction to be executed if a child job + fails due to a reason listed in OnJobFailureReasons. + properties: + action: + description: The action to take if the rule is matched. + enum: + - FailJobSet + - RestartJobSet + - RestartJobSetAndIgnoreMaxRestarts + type: string + name: + description: |- + The name of the failure policy rule. + The name is defaulted to 'failurePolicyRuleN' where N is the index of the failure policy rule. + The name must match the regular expression "^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$". + type: string + onJobFailureReasons: + description: |- + The requirement on the job failure reasons. The requirement + is satisfied if at least one reason matches the list. + The rules are evaluated in order, and the first matching + rule is executed. + An empty list applies the rule to any job failure reason. + items: + type: string + type: array + targetReplicatedJobs: + description: |- + TargetReplicatedJobs are the names of the replicated jobs the operator applies to. + An empty list will apply to all replicatedJobs. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - action + - name + type: object + type: array + type: object + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + managedBy: + description: |- + ManagedBy is used to indicate the controller or entity that manages a JobSet. + The built-in JobSet controller reconciles JobSets which don't have this + field at all or the field value is the reserved string + `jobset.sigs.k8s.io/jobset-controller`, but skips reconciling JobSets + with a custom value for this field. + + The value must be a valid domain-prefixed path (e.g. acme.io/foo) - + all characters before the first "/" must be a valid subdomain as defined + by RFC 1123. All characters trailing the first "/" must be valid HTTP Path + characters as defined by RFC 3986. The value cannot exceed 63 characters. + The field is immutable. + type: string + network: + description: Network defines the networking options for the jobset. + properties: + enableDNSHostnames: + description: |- + EnableDNSHostnames allows pods to be reached via their hostnames. + Pods will be reachable using the fully qualified pod hostname: + ---. + type: boolean + publishNotReadyAddresses: + description: |- + Indicates if DNS records of pods should be published before the pods are ready. + Defaults to True. + type: boolean + subdomain: + description: |- + Subdomain is an explicit choice for a network subdomain name + When set, any replicated job in the set is added to this network. + Defaults to if not set. + type: string + type: object + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + replicatedJobs: + description: ReplicatedJobs is the group of jobs that will form the set. + items: + properties: + name: + description: |- + Name is the name of the entry and will be used as a suffix + for the Job name. + type: string + replicas: + default: 1 + description: |- + Replicas is the number of jobs that will be created from this ReplicatedJob's template. + Jobs names will be in the format: -- + format: int32 + type: integer + template: + description: Template defines the template of the Job that will be created. + properties: + metadata: + description: |- + Standard object's metadata of the jobs created from this template. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + Specification of the desired behavior of the job. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Specifies the duration in seconds relative to the startTime that the job + may be continuously active before the system tries to terminate it; value + must be positive integer. If a Job is suspended (at creation or through an + update), this timer will effectively be stopped and reset when the Job is + resumed again. + format: int64 + type: integer + backoffLimit: + description: |- + Specifies the number of retries before marking this job failed. + Defaults to 6 + format: int32 + type: integer + backoffLimitPerIndex: + description: |- + Specifies the limit for the number of retries within an + index before marking this index as failed. When enabled the number of + failures per index is kept in the pod's + batch.kubernetes.io/job-index-failure-count annotation. It can only + be set when Job's completionMode=Indexed, and the Pod's restart + policy is Never. The field is immutable. + This field is beta-level. It can be used when the `JobBackoffLimitPerIndex` + feature gate is enabled (enabled by default). + format: int32 + type: integer + completionMode: + description: |- + completionMode specifies how Pod completions are tracked. It can be + `NonIndexed` (default) or `Indexed`. + + `NonIndexed` means that the Job is considered complete when there have + been .spec.completions successfully completed Pods. Each Pod completion is + homologous to each other. + + `Indexed` means that the Pods of a + Job get an associated completion index from 0 to (.spec.completions - 1), + available in the annotation batch.kubernetes.io/job-completion-index. + The Job is considered complete when there is one successfully completed Pod + for each index. + When value is `Indexed`, .spec.completions must be specified and + `.spec.parallelism` must be less than or equal to 10^5. + In addition, The Pod name takes the form + `$(job-name)-$(index)-$(random-string)`, + the Pod hostname takes the form `$(job-name)-$(index)`. + + More completion modes can be added in the future. + If the Job controller observes a mode that it doesn't recognize, which + is possible during upgrades due to version skew, the controller + skips updates for the Job. + type: string + completions: + description: |- + Specifies the desired number of successfully finished pods the + job should be run with. Setting to null means that the success of any + pod signals the success of all pods, and allows parallelism to have any positive + value. Setting to 1 means that parallelism is limited to 1 and the success of that + pod signals the success of the job. + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ + format: int32 + type: integer + managedBy: + description: |- + ManagedBy field indicates the controller that manages a Job. The k8s Job + controller reconciles jobs which don't have this field at all or the field + value is the reserved string `kubernetes.io/job-controller`, but skips + reconciling Jobs with a custom value for this field. + The value must be a valid domain-prefixed path (e.g. acme.io/foo) - + all characters before the first "/" must be a valid subdomain as defined + by RFC 1123. All characters trailing the first "/" must be valid HTTP Path + characters as defined by RFC 3986. The value cannot exceed 63 characters. + This field is immutable. + + This field is alpha-level. The job controller accepts setting the field + when the feature gate JobManagedBy is enabled (disabled by default). + type: string + manualSelector: + description: |- + manualSelector controls generation of pod labels and pod selectors. + Leave `manualSelector` unset unless you are certain what you are doing. + When false or unset, the system pick labels unique to this job + and appends those labels to the pod template. When true, + the user is responsible for picking unique labels and specifying + the selector. Failure to pick a unique label may cause this + and other jobs to not function correctly. However, You may see + `manualSelector=true` in jobs that were created with the old `extensions/v1beta1` + API. + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#specifying-your-own-pod-selector + type: boolean + maxFailedIndexes: + description: |- + Specifies the maximal number of failed indexes before marking the Job as + failed, when backoffLimitPerIndex is set. Once the number of failed + indexes exceeds this number the entire Job is marked as Failed and its + execution is terminated. When left as null the job continues execution of + all of its indexes and is marked with the `Complete` Job condition. + It can only be specified when backoffLimitPerIndex is set. + It can be null or up to completions. It is required and must be + less than or equal to 10^4 when is completions greater than 10^5. + This field is beta-level. It can be used when the `JobBackoffLimitPerIndex` + feature gate is enabled (enabled by default). + format: int32 + type: integer + parallelism: + description: |- + Specifies the maximum desired number of pods the job should + run at any given time. The actual number of pods running in steady state will + be less than this number when ((.spec.completions - .status.successful) < .spec.parallelism), + i.e. when the work left to do is less than max parallelism. + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ + format: int32 + type: integer + podFailurePolicy: + description: |- + Specifies the policy of handling failed pods. In particular, it allows to + specify the set of actions and conditions which need to be + satisfied to take the associated action. + If empty, the default behaviour applies - the counter of failed pods, + represented by the jobs's .status.failed field, is incremented and it is + checked against the backoffLimit. This field cannot be used in combination + with restartPolicy=OnFailure. + properties: + rules: + description: |- + A list of pod failure policy rules. The rules are evaluated in order. + Once a rule matches a Pod failure, the remaining of the rules are ignored. + When no rule matches the Pod failure, the default handling applies - the + counter of pod failures is incremented and it is checked against + the backoffLimit. At most 20 elements are allowed. + items: + description: |- + PodFailurePolicyRule describes how a pod failure is handled when the requirements are met. + One of onExitCodes and onPodConditions, but not both, can be used in each rule. + properties: + action: + description: |- + Specifies the action taken on a pod failure when the requirements are satisfied. + Possible values are: + + - FailJob: indicates that the pod's job is marked as Failed and all + running pods are terminated. + - FailIndex: indicates that the pod's index is marked as Failed and will + not be restarted. + This value is beta-level. It can be used when the + `JobBackoffLimitPerIndex` feature gate is enabled (enabled by default). + - Ignore: indicates that the counter towards the .backoffLimit is not + incremented and a replacement pod is created. + - Count: indicates that the pod is handled in the default way - the + counter towards the .backoffLimit is incremented. + Additional values are considered to be added in the future. Clients should + react to an unknown action by skipping the rule. + type: string + onExitCodes: + description: Represents the requirement on the container exit codes. + properties: + containerName: + description: |- + Restricts the check for exit codes to the container with the + specified name. When null, the rule applies to all containers. + When specified, it should match one the container or initContainer + names in the pod template. + type: string + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Containers completed with success (exit code 0) are + excluded from the requirement check. Possible values are: + + - In: the requirement is satisfied if at least one container exit code + (might be multiple if there are multiple containers not restricted + by the 'containerName' field) is in the set of specified values. + - NotIn: the requirement is satisfied if at least one container exit code + (might be multiple if there are multiple containers not restricted + by the 'containerName' field) is not in the set of specified values. + Additional values are considered to be added in the future. Clients should + react to an unknown operator by assuming the requirement is not satisfied. + type: string + values: + description: |- + Specifies the set of values. Each returned container exit code (might be + multiple in case of multiple containers) is checked against this set of + values with respect to the operator. The list of values must be ordered + and must not contain duplicates. Value '0' cannot be used for the In operator. + At least one element is required. At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + - values + type: object + onPodConditions: + description: |- + Represents the requirement on the pod conditions. The requirement is represented + as a list of pod condition patterns. The requirement is satisfied if at + least one pattern matches an actual pod condition. At most 20 elements are allowed. + items: + description: |- + PodFailurePolicyOnPodConditionsPattern describes a pattern for matching + an actual pod condition type. + properties: + status: + description: |- + Specifies the required Pod condition status. To match a pod condition + it is required that the specified status equals the pod condition status. + Defaults to True. + type: string + type: + description: |- + Specifies the required Pod condition type. To match a pod condition + it is required that specified type equals the pod condition type. + type: string + required: + - status + - type + type: object + type: array + x-kubernetes-list-type: atomic + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + required: + - rules + type: object + podReplacementPolicy: + description: |- + podReplacementPolicy specifies when to create replacement Pods. + Possible values are: + - TerminatingOrFailed means that we recreate pods + when they are terminating (has a metadata.deletionTimestamp) or failed. + - Failed means to wait until a previously created Pod is fully terminated (has phase + Failed or Succeeded) before creating a replacement Pod. + + When using podFailurePolicy, Failed is the the only allowed value. + TerminatingOrFailed and Failed are allowed values when podFailurePolicy is not in use. + This is an beta field. To use this, enable the JobPodReplacementPolicy feature toggle. + This is on by default. + type: string + selector: + description: |- + A label query over pods that should match the pod count. + Normally, the system sets this field for you. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + successPolicy: + description: |- + successPolicy specifies the policy when the Job can be declared as succeeded. + If empty, the default behavior applies - the Job is declared as succeeded + only when the number of succeeded pods equals to the completions. + When the field is specified, it must be immutable and works only for the Indexed Jobs. + Once the Job meets the SuccessPolicy, the lingering pods are terminated. + + This field is beta-level. To use this field, you must enable the + `JobSuccessPolicy` feature gate (enabled by default). + properties: + rules: + description: |- + rules represents the list of alternative rules for the declaring the Jobs + as successful before `.status.succeeded >= .spec.completions`. Once any of the rules are met, + the "SucceededCriteriaMet" condition is added, and the lingering pods are removed. + The terminal state for such a Job has the "Complete" condition. + Additionally, these rules are evaluated in order; Once the Job meets one of the rules, + other rules are ignored. At most 20 elements are allowed. + items: + description: |- + SuccessPolicyRule describes rule for declaring a Job as succeeded. + Each rule must have at least one of the "succeededIndexes" or "succeededCount" specified. + properties: + succeededCount: + description: |- + succeededCount specifies the minimal required size of the actual set of the succeeded indexes + for the Job. When succeededCount is used along with succeededIndexes, the check is + constrained only to the set of indexes specified by succeededIndexes. + For example, given that succeededIndexes is "1-4", succeededCount is "3", + and completed indexes are "1", "3", and "5", the Job isn't declared as succeeded + because only "1" and "3" indexes are considered in that rules. + When this field is null, this doesn't default to any value and + is never evaluated at any time. + When specified it needs to be a positive integer. + format: int32 + type: integer + succeededIndexes: + description: |- + succeededIndexes specifies the set of indexes + which need to be contained in the actual set of the succeeded indexes for the Job. + The list of indexes must be within 0 to ".spec.completions-1" and + must not contain duplicates. At least one element is required. + The indexes are represented as intervals separated by commas. + The intervals can be a decimal integer or a pair of decimal integers separated by a hyphen. + The number are listed in represented by the first and last element of the series, + separated by a hyphen. + For example, if the completed indexes are 1, 3, 4, 5 and 7, they are + represented as "1,3-5,7". + When this field is null, this field doesn't default to any value + and is never evaluated at any time. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + required: + - rules + type: object + suspend: + description: |- + suspend specifies whether the Job controller should create Pods or not. If + a Job is created with suspend set to true, no Pods are created by the Job + controller. If a Job is suspended after creation (i.e. the flag goes from + false to true), the Job controller will delete all active Pods associated + with this Job. Users must design their workload to gracefully handle this. + Suspending a Job will reset the StartTime field of the Job, effectively + resetting the ActiveDeadlineSeconds timer too. Defaults to false. + type: boolean + template: + description: |- + Describes the pod that will be created when executing a job. + The only allowed template.spec.restartPolicy values are "Never" or "OnFailure". + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates whether a service account token should be automatically mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container that you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present in a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration that the container should sleep before being terminated. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration that the container should sleep before being terminated. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. + items: + description: volumeDevice describes a mapping of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS resolver options of a pod. + properties: + name: + description: Required. + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present in a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed for ephemeral containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration that the container should sleep before being terminated. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration that the container should sleep before being terminated. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral containers. + items: + description: ContainerPort represents a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + This may only be set for init containers. You cannot set this field on + ephemeral containers. + type: string + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. + items: + description: volumeDevice describes a mapping of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + If this option is set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + of that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container that you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present in a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration that the container should sleep before being terminated. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration that the container should sleep before being terminated. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. + items: + description: volumeDevice describes a mapping of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the reference to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition in the pod's condition list with matching type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated to a Pod to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume in a pod that may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching mode: None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File Service mount on the host and bind mount to the pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret that contains Azure Storage Account Name and Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount on the host that shares a pod's lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as the mounted root, rather than the full Ceph tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents ephemeral storage that is handled by certain external CSI drivers (Beta feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API about the pod that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API volume file + items: + description: DownwardAPIVolumeFile represents information to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field holds extra command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the dataset. This is unique identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx volume attached and mounted on kubelets host machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources secrets, configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the volume root to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about the downwardAPI data to project + properties: + items: + description: Items is a list of DownwardAPIVolume file + items: + description: DownwardAPIVolumeFile represents information to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information about the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify whether the Secret or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information about the serviceAccountToken data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: quobyte represents a Quobyte mount on the host that shares a pod's lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of the ScaleIO Protection Domain for the configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable SSL communication with Gateway, default false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage Pool associated with the protection domain. + type: string + system: + description: system is the name of the storage system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage Policy Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object + ttlSecondsAfterFinished: + description: |- + ttlSecondsAfterFinished limits the lifetime of a Job that has finished + execution (either Complete or Failed). If this field is set, + ttlSecondsAfterFinished after the Job finishes, it is eligible to be + automatically deleted. When the Job is being deleted, its lifecycle + guarantees (e.g. finalizers) will be honored. If this field is unset, + the Job won't be automatically deleted. If this field is set to zero, + the Job becomes eligible to be deleted immediately after it finishes. + format: int32 + type: integer + required: + - template + type: object + type: object + required: + - name + - template + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + startupPolicy: + description: StartupPolicy, if set, configures in what order jobs must be started + properties: + startupPolicyOrder: + description: |- + StartupPolicyOrder determines the startup order of the ReplicatedJobs. + AnyOrder means to start replicated jobs in any order. + InOrder means to start them as they are listed in the JobSet. A ReplicatedJob is started only + when all the jobs of the previous one are ready. + enum: + - AnyOrder + - InOrder + type: string + required: + - startupPolicyOrder + type: object + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + successPolicy: + description: |- + SuccessPolicy configures when to declare the JobSet as + succeeded. + The JobSet is always declared succeeded if all jobs in the set + finished with status complete. + properties: + operator: + description: Operator determines either All or Any of the selected jobs should succeed to consider the JobSet successful + enum: + - All + - Any + type: string + targetReplicatedJobs: + description: |- + TargetReplicatedJobs are the names of the replicated jobs the operator will apply to. + A null or empty list will apply to all replicatedJobs. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - operator + type: object + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + suspend: + description: Suspend suspends all running child Jobs when set to true. + type: boolean + ttlSecondsAfterFinished: + description: |- + TTLSecondsAfterFinished limits the lifetime of a JobSet that has finished + execution (either Complete or Failed). If this field is set, + TTLSecondsAfterFinished after the JobSet finishes, it is eligible to be + automatically deleted. When the JobSet is being deleted, its lifecycle + guarantees (e.g. finalizers) will be honored. If this field is unset, + the JobSet won't be automatically deleted. If this field is set to zero, + the JobSet becomes eligible to be deleted immediately after it finishes. + format: int32 + minimum: 0 + type: integer + type: object + status: + description: JobSetStatus defines the observed state of JobSet + properties: + conditions: + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + replicatedJobsStatus: + description: ReplicatedJobsStatus track the number of JobsReady for each replicatedJob. + items: + description: ReplicatedJobStatus defines the observed ReplicatedJobs Readiness. + properties: + active: + description: |- + Active is the number of child Jobs with at least 1 pod in a running or pending state + which are not marked for deletion. + format: int32 + type: integer + failed: + description: Failed is the number of failed child Jobs. + format: int32 + type: integer + name: + description: Name of the ReplicatedJob. + type: string + ready: + description: |- + Ready is the number of child Jobs where the number of ready pods and completed pods + is greater than or equal to the total expected pod count for the Job (i.e., the minimum + of job.spec.parallelism and job.spec.completions). + format: int32 + type: integer + succeeded: + description: Succeeded is the number of successfully completed child Jobs. + format: int32 + type: integer + suspended: + description: Suspended is the number of child Jobs which are in a suspended state. + format: int32 + type: integer + required: + - active + - failed + - name + - ready + - succeeded + - suspended + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + restarts: + description: Restarts tracks the number of times the JobSet has restarted (i.e. recreated in case of RecreateAll policy). + format: int32 + type: integer + restartsCountTowardsMax: + description: RestartsCountTowardsMax tracks the number of times the JobSet has restarted that counts towards the maximum allowed number of restarts. + format: int32 + type: integer + terminalState: + description: |- + TerminalState the state of the JobSet when it finishes execution. + It can be either Complete or Failed. Otherwise, it is empty by default. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: controller-manager + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: serviceaccount + app.kubernetes.io/part-of: jobset + name: jobset-controller-manager + namespace: jobset-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: leader-election-role + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: role + app.kubernetes.io/part-of: jobset + name: jobset-leader-election-role + namespace: jobset-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: jobset-manager-role +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - get + - list + - update + - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch + resources: + - jobs/status + verbs: + - get + - patch + - update +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/finalizers + verbs: + - update +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/status + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: metrics-reader + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clusterrole + app.kubernetes.io/part-of: jobset + name: jobset-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: proxy-role + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clusterrole + app.kubernetes.io/part-of: jobset + name: jobset-proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: leader-election-rolebinding + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: rolebinding + app.kubernetes.io/part-of: jobset + name: jobset-leader-election-rolebinding + namespace: jobset-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: jobset-leader-election-role +subjects: +- kind: ServiceAccount + name: jobset-controller-manager + namespace: jobset-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: manager-rolebinding + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clusterrolebinding + app.kubernetes.io/part-of: jobset + name: jobset-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: jobset-manager-role +subjects: +- kind: ServiceAccount + name: jobset-controller-manager + namespace: jobset-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: proxy-rolebinding + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clusterrolebinding + app.kubernetes.io/part-of: jobset + name: jobset-proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: jobset-proxy-role +subjects: +- kind: ServiceAccount + name: jobset-controller-manager + namespace: jobset-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: jobset-webhook-server-cert + namespace: jobset-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: controller-manager-metrics-service + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: service + app.kubernetes.io/part-of: jobset + control-plane: controller-manager + name: jobset-controller-manager-metrics-service + namespace: jobset-system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + control-plane: controller-manager +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: webhook-service + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: service + app.kubernetes.io/part-of: jobset + name: jobset-webhook-service + namespace: jobset-system +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: manager + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: controller-manager + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: deployment + app.kubernetes.io/part-of: jobset + control-plane: controller-manager + name: jobset-controller-manager + namespace: jobset-system +spec: + replicas: 1 + selector: + matchLabels: + control-plane: controller-manager + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + control-plane: controller-manager + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - arm64 + - ppc64le + - s390x + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - args: + - --zap-log-level=2 + - --health-probe-bind-address=:8081 + - --metrics-bind-address=127.0.0.1:8080 + - --leader-elect + command: + - /manager + image: registry.k8s.io/jobset/jobset:v0.7.2 + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 2 + memory: 512Mi + requests: + cpu: 500m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + - args: + - --secure-listen-address=0.0.0.0:8443 + - --upstream=http://127.0.0.1:8080/ + - --logtostderr=true + - --v=0 + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1 + name: kube-rbac-proxy + ports: + - containerPort: 8443 + name: https + protocol: TCP + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + securityContext: + runAsNonRoot: true + serviceAccountName: jobset-controller-manager + terminationGracePeriodSeconds: 10 + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: jobset-webhook-server-cert + tolerations: + - effect: NoSchedule + key: components.gke.io/gke-managed-components + operator: Equal + value: "true" +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: jobset-mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: jobset-webhook-service + namespace: jobset-system + path: /mutate-jobset-x-k8s-io-v1alpha2-jobset + failurePolicy: Fail + name: mjobset.kb.io + rules: + - apiGroups: + - jobset.x-k8s.io + apiVersions: + - v1alpha2 + operations: + - CREATE + - UPDATE + resources: + - jobsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: jobset-webhook-service + namespace: jobset-system + path: /mutate--v1-pod + failurePolicy: Fail + name: mpod.kb.io + objectSelector: + matchExpressions: + - key: jobset.sigs.k8s.io/jobset-name + operator: Exists + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: jobset-validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: jobset-webhook-service + namespace: jobset-system + path: /validate-jobset-x-k8s-io-v1alpha2-jobset + failurePolicy: Fail + name: vjobset.kb.io + rules: + - apiGroups: + - jobset.x-k8s.io + apiVersions: + - v1alpha2 + operations: + - CREATE + - UPDATE + resources: + - jobsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: jobset-webhook-service + namespace: jobset-system + path: /validate--v1-pod + failurePolicy: Fail + name: vpod.kb.io + objectSelector: + matchExpressions: + - key: jobset.sigs.k8s.io/jobset-name + operator: Exists + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/jobset-v0.8.1.yaml b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/jobset-v0.8.1.yaml new file mode 100644 index 0000000000..f5b0d82245 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/jobset-v0.8.1.yaml @@ -0,0 +1,9075 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + labels: + app.kubernetes.io/component: manager + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: system + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: namespace + app.kubernetes.io/part-of: jobset + control-plane: controller-manager + name: jobset-system +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + controller-gen.kubebuilder.io/version: v0.16.2 + name: jobsets.jobset.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: jobset-webhook-service + namespace: jobset-system + path: /convert + conversionReviewVersions: + - v1 + group: jobset.x-k8s.io + names: + kind: JobSet + listKind: JobSetList + plural: jobsets + singular: jobset + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Final state of JobSet + jsonPath: .status.terminalState + name: TerminalState + type: string + - description: Number of restarts + jsonPath: .status.restarts + name: Restarts + type: string + - jsonPath: .status.conditions[?(@.type=="Completed")].status + name: Completed + type: string + - description: JobSet suspended + jsonPath: .spec.suspend + name: Suspended + type: string + - description: Time this JobSet was created + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha2 + schema: + openAPIV3Schema: + description: JobSet is the Schema for the jobsets API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: JobSetSpec defines the desired state of JobSet + properties: + coordinator: + description: |- + Coordinator can be used to assign a specific pod as the coordinator for + the JobSet. If defined, an annotation will be added to all Jobs and pods with + coordinator pod, which contains the stable network endpoint where the + coordinator pod can be reached. + jobset.sigs.k8s.io/coordinator=. + properties: + jobIndex: + description: |- + JobIndex is the index of Job which contains the coordinator pod + (i.e., for a ReplicatedJob with N replicas, there are Job indexes 0 to N-1). + type: integer + podIndex: + description: PodIndex is the Job completion index of the coordinator pod. + type: integer + replicatedJob: + description: |- + ReplicatedJob is the name of the ReplicatedJob which contains + the coordinator pod. + type: string + required: + - replicatedJob + type: object + failurePolicy: + description: |- + FailurePolicy, if set, configures when to declare the JobSet as + failed. + The JobSet is always declared failed if any job in the set + finished with status failed. + properties: + maxRestarts: + description: |- + MaxRestarts defines the limit on the number of JobSet restarts. + A restart is achieved by recreating all active child jobs. + format: int32 + type: integer + restartStrategy: + default: Recreate + description: |- + RestartStrategy defines the strategy to use when restarting the JobSet. + Defaults to Recreate. + enum: + - Recreate + - BlockingRecreate + type: string + rules: + description: |- + List of failure policy rules for this JobSet. + For a given Job failure, the rules will be evaluated in order, + and only the first matching rule will be executed. + If no matching rule is found, the RestartJobSet action is applied. + items: + description: |- + FailurePolicyRule defines a FailurePolicyAction to be executed if a child job + fails due to a reason listed in OnJobFailureReasons. + properties: + action: + description: The action to take if the rule is matched. + enum: + - FailJobSet + - RestartJobSet + - RestartJobSetAndIgnoreMaxRestarts + type: string + name: + description: |- + The name of the failure policy rule. + The name is defaulted to 'failurePolicyRuleN' where N is the index of the failure policy rule. + The name must match the regular expression "^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$". + type: string + onJobFailureReasons: + description: |- + The requirement on the job failure reasons. The requirement + is satisfied if at least one reason matches the list. + The rules are evaluated in order, and the first matching + rule is executed. + An empty list applies the rule to any job failure reason. + items: + type: string + type: array + targetReplicatedJobs: + description: |- + TargetReplicatedJobs are the names of the replicated jobs the operator applies to. + An empty list will apply to all replicatedJobs. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - action + - name + type: object + type: array + type: object + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + managedBy: + description: |- + ManagedBy is used to indicate the controller or entity that manages a JobSet. + The built-in JobSet controller reconciles JobSets which don't have this + field at all or the field value is the reserved string + `jobset.sigs.k8s.io/jobset-controller`, but skips reconciling JobSets + with a custom value for this field. + + The value must be a valid domain-prefixed path (e.g. acme.io/foo) - + all characters before the first "/" must be a valid subdomain as defined + by RFC 1123. All characters trailing the first "/" must be valid HTTP Path + characters as defined by RFC 3986. The value cannot exceed 63 characters. + The field is immutable. + type: string + network: + description: Network defines the networking options for the jobset. + properties: + enableDNSHostnames: + description: |- + EnableDNSHostnames allows pods to be reached via their hostnames. + Pods will be reachable using the fully qualified pod hostname: + ---. + type: boolean + publishNotReadyAddresses: + description: |- + Indicates if DNS records of pods should be published before the pods are ready. + Defaults to True. + type: boolean + subdomain: + description: |- + Subdomain is an explicit choice for a network subdomain name + When set, any replicated job in the set is added to this network. + Defaults to if not set. + type: string + type: object + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + replicatedJobs: + description: ReplicatedJobs is the group of jobs that will form the set. + items: + properties: + dependsOn: + description: |- + DependsOn is an optional list that specifies the preceding ReplicatedJobs upon which + the current ReplicatedJob depends. If specified, the ReplicatedJob will be created + only after the referenced ReplicatedJobs reach their desired state. + The Order of ReplicatedJobs is defined by their enumeration in the slice. + Note, that the first ReplicatedJob in the slice cannot use the DependsOn API. + Currently, only a single item is supported in the DependsOn list. + If JobSet is suspended the all active ReplicatedJobs will be suspended. When JobSet is + resumed the Job sequence starts again. + This API is mutually exclusive with the StartupPolicy API. + items: + description: DependsOn defines the dependency on the previous ReplicatedJob status. + properties: + name: + description: Name of the previous ReplicatedJob. + type: string + status: + description: Status defines the condition for the ReplicatedJob. Only Ready or Complete status can be set. + enum: + - Ready + - Complete + type: string + required: + - name + - status + type: object + maxItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + name: + description: |- + Name is the name of the entry and will be used as a suffix + for the Job name. + type: string + replicas: + default: 1 + description: |- + Replicas is the number of jobs that will be created from this ReplicatedJob's template. + Jobs names will be in the format: -- + format: int32 + type: integer + template: + description: Template defines the template of the Job that will be created. + properties: + metadata: + description: |- + Standard object's metadata of the jobs created from this template. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + Specification of the desired behavior of the job. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Specifies the duration in seconds relative to the startTime that the job + may be continuously active before the system tries to terminate it; value + must be positive integer. If a Job is suspended (at creation or through an + update), this timer will effectively be stopped and reset when the Job is + resumed again. + format: int64 + type: integer + backoffLimit: + description: |- + Specifies the number of retries before marking this job failed. + Defaults to 6 + format: int32 + type: integer + backoffLimitPerIndex: + description: |- + Specifies the limit for the number of retries within an + index before marking this index as failed. When enabled the number of + failures per index is kept in the pod's + batch.kubernetes.io/job-index-failure-count annotation. It can only + be set when Job's completionMode=Indexed, and the Pod's restart + policy is Never. The field is immutable. + This field is beta-level. It can be used when the `JobBackoffLimitPerIndex` + feature gate is enabled (enabled by default). + format: int32 + type: integer + completionMode: + description: |- + completionMode specifies how Pod completions are tracked. It can be + `NonIndexed` (default) or `Indexed`. + + `NonIndexed` means that the Job is considered complete when there have + been .spec.completions successfully completed Pods. Each Pod completion is + homologous to each other. + + `Indexed` means that the Pods of a + Job get an associated completion index from 0 to (.spec.completions - 1), + available in the annotation batch.kubernetes.io/job-completion-index. + The Job is considered complete when there is one successfully completed Pod + for each index. + When value is `Indexed`, .spec.completions must be specified and + `.spec.parallelism` must be less than or equal to 10^5. + In addition, The Pod name takes the form + `$(job-name)-$(index)-$(random-string)`, + the Pod hostname takes the form `$(job-name)-$(index)`. + + More completion modes can be added in the future. + If the Job controller observes a mode that it doesn't recognize, which + is possible during upgrades due to version skew, the controller + skips updates for the Job. + type: string + completions: + description: |- + Specifies the desired number of successfully finished pods the + job should be run with. Setting to null means that the success of any + pod signals the success of all pods, and allows parallelism to have any positive + value. Setting to 1 means that parallelism is limited to 1 and the success of that + pod signals the success of the job. + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ + format: int32 + type: integer + managedBy: + description: |- + ManagedBy field indicates the controller that manages a Job. The k8s Job + controller reconciles jobs which don't have this field at all or the field + value is the reserved string `kubernetes.io/job-controller`, but skips + reconciling Jobs with a custom value for this field. + The value must be a valid domain-prefixed path (e.g. acme.io/foo) - + all characters before the first "/" must be a valid subdomain as defined + by RFC 1123. All characters trailing the first "/" must be valid HTTP Path + characters as defined by RFC 3986. The value cannot exceed 63 characters. + This field is immutable. + + This field is beta-level. The job controller accepts setting the field + when the feature gate JobManagedBy is enabled (enabled by default). + type: string + manualSelector: + description: |- + manualSelector controls generation of pod labels and pod selectors. + Leave `manualSelector` unset unless you are certain what you are doing. + When false or unset, the system pick labels unique to this job + and appends those labels to the pod template. When true, + the user is responsible for picking unique labels and specifying + the selector. Failure to pick a unique label may cause this + and other jobs to not function correctly. However, You may see + `manualSelector=true` in jobs that were created with the old `extensions/v1beta1` + API. + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#specifying-your-own-pod-selector + type: boolean + maxFailedIndexes: + description: |- + Specifies the maximal number of failed indexes before marking the Job as + failed, when backoffLimitPerIndex is set. Once the number of failed + indexes exceeds this number the entire Job is marked as Failed and its + execution is terminated. When left as null the job continues execution of + all of its indexes and is marked with the `Complete` Job condition. + It can only be specified when backoffLimitPerIndex is set. + It can be null or up to completions. It is required and must be + less than or equal to 10^4 when is completions greater than 10^5. + This field is beta-level. It can be used when the `JobBackoffLimitPerIndex` + feature gate is enabled (enabled by default). + format: int32 + type: integer + parallelism: + description: |- + Specifies the maximum desired number of pods the job should + run at any given time. The actual number of pods running in steady state will + be less than this number when ((.spec.completions - .status.successful) < .spec.parallelism), + i.e. when the work left to do is less than max parallelism. + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ + format: int32 + type: integer + podFailurePolicy: + description: |- + Specifies the policy of handling failed pods. In particular, it allows to + specify the set of actions and conditions which need to be + satisfied to take the associated action. + If empty, the default behaviour applies - the counter of failed pods, + represented by the jobs's .status.failed field, is incremented and it is + checked against the backoffLimit. This field cannot be used in combination + with restartPolicy=OnFailure. + properties: + rules: + description: |- + A list of pod failure policy rules. The rules are evaluated in order. + Once a rule matches a Pod failure, the remaining of the rules are ignored. + When no rule matches the Pod failure, the default handling applies - the + counter of pod failures is incremented and it is checked against + the backoffLimit. At most 20 elements are allowed. + items: + description: |- + PodFailurePolicyRule describes how a pod failure is handled when the requirements are met. + One of onExitCodes and onPodConditions, but not both, can be used in each rule. + properties: + action: + description: |- + Specifies the action taken on a pod failure when the requirements are satisfied. + Possible values are: + + - FailJob: indicates that the pod's job is marked as Failed and all + running pods are terminated. + - FailIndex: indicates that the pod's index is marked as Failed and will + not be restarted. + This value is beta-level. It can be used when the + `JobBackoffLimitPerIndex` feature gate is enabled (enabled by default). + - Ignore: indicates that the counter towards the .backoffLimit is not + incremented and a replacement pod is created. + - Count: indicates that the pod is handled in the default way - the + counter towards the .backoffLimit is incremented. + Additional values are considered to be added in the future. Clients should + react to an unknown action by skipping the rule. + type: string + onExitCodes: + description: Represents the requirement on the container exit codes. + properties: + containerName: + description: |- + Restricts the check for exit codes to the container with the + specified name. When null, the rule applies to all containers. + When specified, it should match one the container or initContainer + names in the pod template. + type: string + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Containers completed with success (exit code 0) are + excluded from the requirement check. Possible values are: + + - In: the requirement is satisfied if at least one container exit code + (might be multiple if there are multiple containers not restricted + by the 'containerName' field) is in the set of specified values. + - NotIn: the requirement is satisfied if at least one container exit code + (might be multiple if there are multiple containers not restricted + by the 'containerName' field) is not in the set of specified values. + Additional values are considered to be added in the future. Clients should + react to an unknown operator by assuming the requirement is not satisfied. + type: string + values: + description: |- + Specifies the set of values. Each returned container exit code (might be + multiple in case of multiple containers) is checked against this set of + values with respect to the operator. The list of values must be ordered + and must not contain duplicates. Value '0' cannot be used for the In operator. + At least one element is required. At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + - values + type: object + onPodConditions: + description: |- + Represents the requirement on the pod conditions. The requirement is represented + as a list of pod condition patterns. The requirement is satisfied if at + least one pattern matches an actual pod condition. At most 20 elements are allowed. + items: + description: |- + PodFailurePolicyOnPodConditionsPattern describes a pattern for matching + an actual pod condition type. + properties: + status: + description: |- + Specifies the required Pod condition status. To match a pod condition + it is required that the specified status equals the pod condition status. + Defaults to True. + type: string + type: + description: |- + Specifies the required Pod condition type. To match a pod condition + it is required that specified type equals the pod condition type. + type: string + required: + - status + - type + type: object + type: array + x-kubernetes-list-type: atomic + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + required: + - rules + type: object + podReplacementPolicy: + description: |- + podReplacementPolicy specifies when to create replacement Pods. + Possible values are: + - TerminatingOrFailed means that we recreate pods + when they are terminating (has a metadata.deletionTimestamp) or failed. + - Failed means to wait until a previously created Pod is fully terminated (has phase + Failed or Succeeded) before creating a replacement Pod. + + When using podFailurePolicy, Failed is the the only allowed value. + TerminatingOrFailed and Failed are allowed values when podFailurePolicy is not in use. + This is an beta field. To use this, enable the JobPodReplacementPolicy feature toggle. + This is on by default. + type: string + selector: + description: |- + A label query over pods that should match the pod count. + Normally, the system sets this field for you. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + successPolicy: + description: |- + successPolicy specifies the policy when the Job can be declared as succeeded. + If empty, the default behavior applies - the Job is declared as succeeded + only when the number of succeeded pods equals to the completions. + When the field is specified, it must be immutable and works only for the Indexed Jobs. + Once the Job meets the SuccessPolicy, the lingering pods are terminated. + + This field is beta-level. To use this field, you must enable the + `JobSuccessPolicy` feature gate (enabled by default). + properties: + rules: + description: |- + rules represents the list of alternative rules for the declaring the Jobs + as successful before `.status.succeeded >= .spec.completions`. Once any of the rules are met, + the "SucceededCriteriaMet" condition is added, and the lingering pods are removed. + The terminal state for such a Job has the "Complete" condition. + Additionally, these rules are evaluated in order; Once the Job meets one of the rules, + other rules are ignored. At most 20 elements are allowed. + items: + description: |- + SuccessPolicyRule describes rule for declaring a Job as succeeded. + Each rule must have at least one of the "succeededIndexes" or "succeededCount" specified. + properties: + succeededCount: + description: |- + succeededCount specifies the minimal required size of the actual set of the succeeded indexes + for the Job. When succeededCount is used along with succeededIndexes, the check is + constrained only to the set of indexes specified by succeededIndexes. + For example, given that succeededIndexes is "1-4", succeededCount is "3", + and completed indexes are "1", "3", and "5", the Job isn't declared as succeeded + because only "1" and "3" indexes are considered in that rules. + When this field is null, this doesn't default to any value and + is never evaluated at any time. + When specified it needs to be a positive integer. + format: int32 + type: integer + succeededIndexes: + description: |- + succeededIndexes specifies the set of indexes + which need to be contained in the actual set of the succeeded indexes for the Job. + The list of indexes must be within 0 to ".spec.completions-1" and + must not contain duplicates. At least one element is required. + The indexes are represented as intervals separated by commas. + The intervals can be a decimal integer or a pair of decimal integers separated by a hyphen. + The number are listed in represented by the first and last element of the series, + separated by a hyphen. + For example, if the completed indexes are 1, 3, 4, 5 and 7, they are + represented as "1,3-5,7". + When this field is null, this field doesn't default to any value + and is never evaluated at any time. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + required: + - rules + type: object + suspend: + description: |- + suspend specifies whether the Job controller should create Pods or not. If + a Job is created with suspend set to true, no Pods are created by the Job + controller. If a Job is suspended after creation (i.e. the flag goes from + false to true), the Job controller will delete all active Pods associated + with this Job. Users must design their workload to gracefully handle this. + Suspending a Job will reset the StartTime field of the Job, effectively + resetting the ActiveDeadlineSeconds timer too. Defaults to false. + type: boolean + template: + description: |- + Describes the pod that will be created when executing a job. + The only allowed template.spec.restartPolicy values are "Never" or "OnFailure". + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates whether a service account token should be automatically mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container that you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present in a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. + items: + description: volumeDevice describes a mapping of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS resolver options of a pod. + properties: + name: + description: |- + Name is this DNS resolver option's name. + Required. + type: string + value: + description: Value is this DNS resolver option's value. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present in a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed for ephemeral containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral containers. + items: + description: ContainerPort represents a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + This may only be set for init containers. You cannot set this field on + ephemeral containers. + type: string + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for ephemeral containers. + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. + items: + description: volumeDevice describes a mapping of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + If this option is set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + of that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container that you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present in a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection to a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be used by the container. + items: + description: volumeDevice describes a mapping of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the reference to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition in the pod's condition list with matching type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + resources: + description: |- + Resources is the total amount of CPU and Memory resources required by all + containers in the pod. It supports specifying Requests and Limits for + "cpu" and "memory" resource names only. ResourceClaims are not supported. + + This field enables fine-grained control over resource allocation for the + entire pod, allowing resource sharing among containers in a pod. + + This is an alpha field and requires enabling the PodLevelResources feature + gate. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated to a Pod to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume in a pod that may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + Deprecated: AWSElasticBlockStore is deprecated. All operations for the in-tree + awsElasticBlockStore type are redirected to the ebs.csi.aws.com CSI driver. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: |- + azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. + Deprecated: AzureDisk is deprecated. All operations for the in-tree azureDisk type + are redirected to the disk.csi.azure.com CSI driver. + properties: + cachingMode: + description: 'cachingMode is the Host Caching mode: None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: |- + azureFile represents an Azure File Service mount on the host and bind mount to the pod. + Deprecated: AzureFile is deprecated. All operations for the in-tree azureFile type + are redirected to the file.csi.azure.com CSI driver. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret that contains Azure Storage Account Name and Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: |- + cephFS represents a Ceph FS mount on the host that shares a pod's lifetime. + Deprecated: CephFS is deprecated and the in-tree cephfs type is no longer supported. + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as the mounted root, rather than the full Ceph tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + Deprecated: Cinder is deprecated. All operations for the in-tree cinder type + are redirected to the cinder.csi.openstack.org CSI driver. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents ephemeral storage that is handled by certain external CSI drivers. + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API about the pod that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API volume file + items: + description: DownwardAPIVolumeFile represents information to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + Deprecated: FlexVolume is deprecated. Consider using a CSIDriver instead. + properties: + driver: + description: driver is the name of the driver to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field holds extra command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: |- + flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running. + Deprecated: Flocker is deprecated and the in-tree flocker type is no longer supported. + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the dataset. This is unique identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + Deprecated: GCEPersistentDisk is deprecated. All operations for the in-tree + gcePersistentDisk type are redirected to the pd.csi.storage.gke.io CSI driver. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + Deprecated: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: |- + photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine. + Deprecated: PhotonPersistentDisk is deprecated and the in-tree photonPersistentDisk type is no longer supported. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: |- + portworxVolume represents a portworx volume attached and mounted on kubelets host machine. + Deprecated: PortworxVolume is deprecated. All operations for the in-tree portworxVolume type + are redirected to the pxd.portworx.com CSI driver when the CSIMigrationPortworx feature-gate + is on. + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources secrets, configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the volume root to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about the downwardAPI data to project + properties: + items: + description: Items is a list of DownwardAPIVolume file + items: + description: DownwardAPIVolumeFile represents information to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information about the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify whether the Secret or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information about the serviceAccountToken data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: |- + quobyte represents a Quobyte mount on the host that shares a pod's lifetime. + Deprecated: Quobyte is deprecated and the in-tree quobyte type is no longer supported. + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: |- + scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. + Deprecated: ScaleIO is deprecated and the in-tree scaleIO type is no longer supported. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of the ScaleIO Protection Domain for the configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable SSL communication with Gateway, default false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage Pool associated with the protection domain. + type: string + system: + description: system is the name of the storage system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: |- + storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. + Deprecated: StorageOS is deprecated and the in-tree storageos type is no longer supported. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: |- + vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine. + Deprecated: VsphereVolume is deprecated. All operations for the in-tree vsphereVolume type + are redirected to the csi.vsphere.vmware.com CSI driver. + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage Policy Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object + ttlSecondsAfterFinished: + description: |- + ttlSecondsAfterFinished limits the lifetime of a Job that has finished + execution (either Complete or Failed). If this field is set, + ttlSecondsAfterFinished after the Job finishes, it is eligible to be + automatically deleted. When the Job is being deleted, its lifecycle + guarantees (e.g. finalizers) will be honored. If this field is unset, + the Job won't be automatically deleted. If this field is set to zero, + the Job becomes eligible to be deleted immediately after it finishes. + format: int32 + type: integer + required: + - template + type: object + type: object + required: + - name + - template + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + startupPolicy: + description: |- + StartupPolicy, if set, configures in what order jobs must be started + Deprecated: StartupPolicy is deprecated, please use the DependsOn API. + properties: + startupPolicyOrder: + description: |- + StartupPolicyOrder determines the startup order of the ReplicatedJobs. + AnyOrder means to start replicated jobs in any order. + InOrder means to start them as they are listed in the JobSet. A ReplicatedJob is started only + when all the jobs of the previous one are ready. + enum: + - AnyOrder + - InOrder + type: string + required: + - startupPolicyOrder + type: object + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + successPolicy: + description: |- + SuccessPolicy configures when to declare the JobSet as + succeeded. + The JobSet is always declared succeeded if all jobs in the set + finished with status complete. + properties: + operator: + description: Operator determines either All or Any of the selected jobs should succeed to consider the JobSet successful + enum: + - All + - Any + type: string + targetReplicatedJobs: + description: |- + TargetReplicatedJobs are the names of the replicated jobs the operator will apply to. + A null or empty list will apply to all replicatedJobs. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - operator + type: object + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + suspend: + description: Suspend suspends all running child Jobs when set to true. + type: boolean + ttlSecondsAfterFinished: + description: |- + TTLSecondsAfterFinished limits the lifetime of a JobSet that has finished + execution (either Complete or Failed). If this field is set, + TTLSecondsAfterFinished after the JobSet finishes, it is eligible to be + automatically deleted. When the JobSet is being deleted, its lifecycle + guarantees (e.g. finalizers) will be honored. If this field is unset, + the JobSet won't be automatically deleted. If this field is set to zero, + the JobSet becomes eligible to be deleted immediately after it finishes. + format: int32 + minimum: 0 + type: integer + type: object + x-kubernetes-validations: + - message: StartupPolicy and DependsOn APIs are mutually exclusive + rule: '!(has(self.startupPolicy) && self.startupPolicy.startupPolicyOrder == ''InOrder'' && self.replicatedJobs.exists(x, has(x.dependsOn)))' + status: + description: JobSetStatus defines the observed state of JobSet + properties: + conditions: + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + replicatedJobsStatus: + description: ReplicatedJobsStatus track the number of JobsReady for each replicatedJob. + items: + description: ReplicatedJobStatus defines the observed ReplicatedJobs Readiness. + properties: + active: + description: |- + Active is the number of child Jobs with at least 1 pod in a running or pending state + which are not marked for deletion. + format: int32 + type: integer + failed: + description: Failed is the number of failed child Jobs. + format: int32 + type: integer + name: + description: Name of the ReplicatedJob. + type: string + ready: + description: |- + Ready is the number of child Jobs where the number of ready pods and completed pods + is greater than or equal to the total expected pod count for the Job (i.e., the minimum + of job.spec.parallelism and job.spec.completions). + format: int32 + type: integer + succeeded: + description: Succeeded is the number of successfully completed child Jobs. + format: int32 + type: integer + suspended: + description: Suspended is the number of child Jobs which are in a suspended state. + format: int32 + type: integer + required: + - active + - failed + - name + - ready + - succeeded + - suspended + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + restarts: + description: Restarts tracks the number of times the JobSet has restarted (i.e. recreated in case of RecreateAll policy). + format: int32 + type: integer + restartsCountTowardsMax: + description: RestartsCountTowardsMax tracks the number of times the JobSet has restarted that counts towards the maximum allowed number of restarts. + format: int32 + type: integer + terminalState: + description: |- + TerminalState the state of the JobSet when it finishes execution. + It can be either Complete or Failed. Otherwise, it is empty by default. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: controller-manager + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: serviceaccount + app.kubernetes.io/part-of: jobset + name: jobset-controller-manager + namespace: jobset-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: leader-election-role + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: role + app.kubernetes.io/part-of: jobset + name: jobset-leader-election-role + namespace: jobset-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: jobset-manager-role +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - get + - list + - update + - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch + resources: + - jobs/status + verbs: + - get + - patch + - update +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/finalizers + verbs: + - update +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/status + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: metrics-reader + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clusterrole + app.kubernetes.io/part-of: jobset + name: jobset-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: proxy-role + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clusterrole + app.kubernetes.io/part-of: jobset + name: jobset-proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: leader-election-rolebinding + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: rolebinding + app.kubernetes.io/part-of: jobset + name: jobset-leader-election-rolebinding + namespace: jobset-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: jobset-leader-election-role +subjects: +- kind: ServiceAccount + name: jobset-controller-manager + namespace: jobset-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: manager-rolebinding + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clusterrolebinding + app.kubernetes.io/part-of: jobset + name: jobset-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: jobset-manager-role +subjects: +- kind: ServiceAccount + name: jobset-controller-manager + namespace: jobset-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: jobset-metrics-reader-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: jobset-metrics-reader +subjects: +- kind: ServiceAccount + name: jobset-controller-manager + namespace: jobset-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: proxy-rolebinding + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clusterrolebinding + app.kubernetes.io/part-of: jobset + name: jobset-proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: jobset-proxy-role +subjects: +- kind: ServiceAccount + name: jobset-controller-manager + namespace: jobset-system +--- +apiVersion: v1 +data: + controller_manager_config.yaml: |2 + + apiVersion: config.jobset.x-k8s.io/v1alpha1 + kind: Configuration + leaderElection: + leaderElect: true +kind: ConfigMap +metadata: + name: jobset-manager-config + namespace: jobset-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: jobset-webhook-server-cert + namespace: jobset-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: jobset + control-plane: controller-manager + name: jobset-controller-manager-metrics-service + namespace: jobset-system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + control-plane: controller-manager +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: webhook-service + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: service + app.kubernetes.io/part-of: jobset + name: jobset-webhook-service + namespace: jobset-system +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: manager + app.kubernetes.io/created-by: jobset + app.kubernetes.io/instance: controller-manager + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: deployment + app.kubernetes.io/part-of: jobset + control-plane: controller-manager + name: jobset-controller-manager + namespace: jobset-system +spec: + replicas: 1 + selector: + matchLabels: + control-plane: controller-manager + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + control-plane: controller-manager + spec: + containers: + - args: + - --config=/controller_manager_config.yaml + - --zap-log-level=2 + command: + - /manager + image: registry.k8s.io/jobset/jobset:v0.8.1 + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: manager + ports: + - containerPort: 8443 + name: metrics + protocol: TCP + - containerPort: 9443 + name: webhook-server + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + memory: 4Gi + requests: + cpu: 500m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - mountPath: /controller_manager_config.yaml + name: manager-config + subPath: controller_manager_config.yaml + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + securityContext: + runAsNonRoot: true + serviceAccountName: jobset-controller-manager + terminationGracePeriodSeconds: 10 + volumes: + - configMap: + name: jobset-manager-config + name: manager-config + - name: cert + secret: + defaultMode: 420 + secretName: jobset-webhook-server-cert + tolerations: + - effect: NoSchedule + key: components.gke.io/gke-managed-components + operator: Equal + value: "true" +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: jobset-mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: jobset-webhook-service + namespace: jobset-system + path: /mutate-jobset-x-k8s-io-v1alpha2-jobset + failurePolicy: Fail + name: mjobset.kb.io + rules: + - apiGroups: + - jobset.x-k8s.io + apiVersions: + - v1alpha2 + operations: + - CREATE + - UPDATE + resources: + - jobsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: jobset-webhook-service + namespace: jobset-system + path: /mutate--v1-pod + failurePolicy: Fail + name: mpod.kb.io + objectSelector: + matchExpressions: + - key: jobset.sigs.k8s.io/jobset-name + operator: Exists + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: jobset-validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: jobset-webhook-service + namespace: jobset-system + path: /validate-jobset-x-k8s-io-v1alpha2-jobset + failurePolicy: Fail + name: vjobset.kb.io + rules: + - apiGroups: + - jobset.x-k8s.io + apiVersions: + - v1alpha2 + operations: + - CREATE + - UPDATE + resources: + - jobsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: jobset-webhook-service + namespace: jobset-system + path: /validate--v1-pod + failurePolicy: Fail + name: vpod.kb.io + objectSelector: + matchExpressions: + - key: jobset.sigs.k8s.io/jobset-name + operator: Exists + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.10.0.yaml b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.10.0.yaml new file mode 100644 index 0000000000..8fb5db3638 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.10.0.yaml @@ -0,0 +1,13185 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-system +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: admissionchecks.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: AdmissionCheck + listKind: AdmissionCheckList + plural: admissionchecks + singular: admissioncheck + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: AdmissionCheck is the Schema for the admissionchecks API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: AdmissionCheckSpec defines the desired state of AdmissionCheck + properties: + controllerName: + description: |- + controllerName identifies the controller that processes the AdmissionCheck, + not necessarily a Kubernetes Pod or Deployment name. Cannot be empty. + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + parameters: + description: |- + Parameters identifies a configuration with additional parameters for the + check. + properties: + apiGroup: + description: ApiGroup is the group for the resource being referenced. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is the type of the resource being referenced. + maxLength: 63 + pattern: ^(?i)[a-z]([-a-z0-9]*[a-z0-9])?$ + type: string + name: + description: Name is the name of the resource being referenced. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - apiGroup + - kind + - name + type: object + retryDelayMinutes: + default: 15 + description: |- + RetryDelayMinutes specifies how long to keep the workload suspended after + a failed check (after it transitioned to False). When the delay period has passed, the check + state goes to "Unknown". The default is 15 min. + Deprecated: retryDelayMinutes has already been deprecated since v0.8 and will be removed in v1beta2. + format: int64 + type: integer + required: + - controllerName + type: object + status: + description: AdmissionCheckStatus defines the observed state of AdmissionCheck + properties: + conditions: + description: |- + conditions hold the latest available observations of the AdmissionCheck + current state. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: clusterqueues.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: ClusterQueue + listKind: ClusterQueueList + plural: clusterqueues + shortNames: + - cq + singular: clusterqueue + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Cohort that this ClusterQueue belongs to + jsonPath: .spec.cohort + name: Cohort + type: string + - description: The queueing strategy used to prioritize workloads + jsonPath: .spec.queueingStrategy + name: Strategy + priority: 1 + type: string + - description: Number of pending workloads + jsonPath: .status.pendingWorkloads + name: Pending Workloads + type: integer + - description: Number of admitted workloads that haven't finished yet + jsonPath: .status.admittedWorkloads + name: Admitted Workloads + priority: 1 + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + description: ClusterQueue is the Schema for the clusterQueue API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ClusterQueueSpec defines the desired state of ClusterQueue + properties: + admissionChecks: + description: |- + admissionChecks lists the AdmissionChecks required by this ClusterQueue. + Cannot be used along with AdmissionCheckStrategy. + items: + type: string + type: array + admissionChecksStrategy: + description: |- + admissionCheckStrategy defines a list of strategies to determine which ResourceFlavors require AdmissionChecks. + This property cannot be used in conjunction with the 'admissionChecks' property. + properties: + admissionChecks: + description: admissionChecks is a list of strategies for AdmissionChecks + items: + description: AdmissionCheckStrategyRule defines rules for a + single AdmissionCheck + properties: + name: + description: name is an AdmissionCheck's name. + type: string + onFlavors: + description: |- + onFlavors is a list of ResourceFlavors' names that this AdmissionCheck should run for. + If empty, the AdmissionCheck will run for all workloads submitted to the ClusterQueue. + items: + description: ResourceFlavorReference is the name of the + ResourceFlavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + type: array + required: + - name + type: object + type: array + type: object + cohort: + description: |- + cohort that this ClusterQueue belongs to. CQs that belong to the + same cohort can borrow unused resources from each other. + + A CQ can be a member of a single borrowing cohort. A workload submitted + to a queue referencing this CQ can borrow quota from any CQ in the cohort. + Only quota for the [resource, flavor] pairs listed in the CQ can be + borrowed. + If empty, this ClusterQueue cannot borrow from any other ClusterQueue and + vice versa. + + A cohort is a name that links CQs together, but it doesn't reference any + object. + + Validation of a cohort name is equivalent to that of object names: + subdomain in DNS (RFC 1123). + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + fairSharing: + description: |- + fairSharing defines the properties of the ClusterQueue when participating in fair sharing. + The values are only relevant if fair sharing is enabled in the Kueue configuration. + properties: + weight: + anyOf: + - type: integer + - type: string + default: 1 + description: |- + weight gives a comparative advantage to this ClusterQueue when competing for unused + resources in the cohort against other ClusterQueues. + The share of a ClusterQueue is based on the dominant resource usage above nominal + quotas for each resource, divided by the weight. + Admission prioritizes scheduling workloads from ClusterQueues with the lowest share + and preempting workloads from the ClusterQueues with the highest share. + A zero weight implies infinite share value, meaning that this ClusterQueue will always + be at disadvantage against other ClusterQueues. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + flavorFungibility: + default: {} + description: |- + flavorFungibility defines whether a workload should try the next flavor + before borrowing or preempting in the flavor being evaluated. + properties: + whenCanBorrow: + default: Borrow + description: |- + whenCanBorrow determines whether a workload should try the next flavor + before borrowing in current flavor. The possible values are: + + - `Borrow` (default): allocate in current flavor if borrowing + is possible. + - `TryNextFlavor`: try next flavor even if the current + flavor has enough resources to borrow. + enum: + - Borrow + - TryNextFlavor + type: string + whenCanPreempt: + default: TryNextFlavor + description: |- + whenCanPreempt determines whether a workload should try the next flavor + before borrowing in current flavor. The possible values are: + + - `Preempt`: allocate in current flavor if it's possible to preempt some workloads. + - `TryNextFlavor` (default): try next flavor even if there are enough + candidates for preemption in the current flavor. + enum: + - Preempt + - TryNextFlavor + type: string + type: object + namespaceSelector: + description: |- + namespaceSelector defines which namespaces are allowed to submit workloads to + this clusterQueue. Beyond this basic support for policy, a policy agent like + Gatekeeper should be used to enforce more advanced policies. + Defaults to null which is a nothing selector (no namespaces eligible). + If set to an empty selector `{}`, then all namespaces are eligible. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + preemption: + default: {} + description: |- + preemption describes policies to preempt Workloads from this ClusterQueue + or the ClusterQueue's cohort. + + Preemption can happen in two scenarios: + + - When a Workload fits within the nominal quota of the ClusterQueue, but + the quota is currently borrowed by other ClusterQueues in the cohort. + Preempting Workloads in other ClusterQueues allows this ClusterQueue to + reclaim its nominal quota. + - When a Workload doesn't fit within the nominal quota of the ClusterQueue + and there are admitted Workloads in the ClusterQueue with lower priority. + + The preemption algorithm tries to find a minimal set of Workloads to + preempt to accomomdate the pending Workload, preempting Workloads with + lower priority first. + properties: + borrowWithinCohort: + default: {} + description: |- + borrowWithinCohort provides configuration to allow preemption within + cohort while borrowing. + properties: + maxPriorityThreshold: + description: |- + maxPriorityThreshold allows to restrict the set of workloads which + might be preempted by a borrowing workload, to only workloads with + priority less than or equal to the specified threshold priority. + When the threshold is not specified, then any workload satisfying the + policy can be preempted by the borrowing workload. + format: int32 + type: integer + policy: + default: Never + description: |- + policy determines the policy for preemption to reclaim quota within cohort while borrowing. + Possible values are: + - `Never` (default): do not allow for preemption, in other + ClusterQueues within the cohort, for a borrowing workload. + - `LowerPriority`: allow preemption, in other ClusterQueues + within the cohort, for a borrowing workload, but only if + the preempted workloads are of lower priority. + enum: + - Never + - LowerPriority + type: string + type: object + reclaimWithinCohort: + default: Never + description: |- + reclaimWithinCohort determines whether a pending Workload can preempt + Workloads from other ClusterQueues in the cohort that are using more than + their nominal quota. The possible values are: + + - `Never` (default): do not preempt Workloads in the cohort. + - `LowerPriority`: **Classic Preemption** if the pending Workload + fits within the nominal quota of its ClusterQueue, only preempt + Workloads in the cohort that have lower priority than the pending + Workload. **Fair Sharing** only preempt Workloads in the cohort that + have lower priority than the pending Workload and that satisfy the + fair sharing preemptionStategies. + - `Any`: **Classic Preemption** if the pending Workload fits within + the nominal quota of its ClusterQueue, preempt any Workload in the + cohort, irrespective of priority. **Fair Sharing** preempt Workloads + in the cohort that satisfy the fair sharing preemptionStrategies. + enum: + - Never + - LowerPriority + - Any + type: string + withinClusterQueue: + default: Never + description: |- + withinClusterQueue determines whether a pending Workload that doesn't fit + within the nominal quota for its ClusterQueue, can preempt active Workloads in + the ClusterQueue. The possible values are: + + - `Never` (default): do not preempt Workloads in the ClusterQueue. + - `LowerPriority`: only preempt Workloads in the ClusterQueue that have + lower priority than the pending Workload. + - `LowerOrNewerEqualPriority`: only preempt Workloads in the ClusterQueue that + either have a lower priority than the pending workload or equal priority + and are newer than the pending workload. + enum: + - Never + - LowerPriority + - LowerOrNewerEqualPriority + type: string + type: object + x-kubernetes-validations: + - message: reclaimWithinCohort=Never and borrowWithinCohort.Policy!=Never + rule: '!(self.reclaimWithinCohort == ''Never'' && has(self.borrowWithinCohort) + && self.borrowWithinCohort.policy != ''Never'')' + queueingStrategy: + default: BestEffortFIFO + description: |- + QueueingStrategy indicates the queueing strategy of the workloads + across the queues in this ClusterQueue. + Current Supported Strategies: + + - StrictFIFO: workloads are ordered strictly by creation time. + Older workloads that can't be admitted will block admitting newer + workloads even if they fit available quota. + - BestEffortFIFO: workloads are ordered by creation time, + however older workloads that can't be admitted will not block + admitting newer workloads that fit existing quota. + enum: + - StrictFIFO + - BestEffortFIFO + type: string + resourceGroups: + description: |- + resourceGroups describes groups of resources. + Each resource group defines the list of resources and a list of flavors + that provide quotas for these resources. + Each resource and each flavor can only form part of one resource group. + resourceGroups can be up to 16. + items: + properties: + coveredResources: + description: |- + coveredResources is the list of resources covered by the flavors in this + group. + Examples: cpu, memory, vendor.com/gpu. + The list cannot be empty and it can contain up to 16 resources. + items: + description: ResourceName is the name identifying various + resources in a ResourceList. + type: string + maxItems: 16 + minItems: 1 + type: array + flavors: + description: |- + flavors is the list of flavors that provide the resources of this group. + Typically, different flavors represent different hardware models + (e.g., gpu models, cpu architectures) or pricing models (on-demand vs spot + cpus). + Each flavor MUST list all the resources listed for this group in the same + order as the .resources field. + The list cannot be empty and it can contain up to 16 flavors. + items: + properties: + name: + description: |- + name of this flavor. The name should match the .metadata.name of a + ResourceFlavor. If a matching ResourceFlavor does not exist, the + ClusterQueue will have an Active condition set to False. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: |- + resources is the list of quotas for this flavor per resource. + There could be up to 16 resources. + items: + properties: + borrowingLimit: + anyOf: + - type: integer + - type: string + description: |- + borrowingLimit is the maximum amount of quota for the [flavor, resource] + combination that this ClusterQueue is allowed to borrow from the unused + quota of other ClusterQueues in the same cohort. + In total, at a given time, Workloads in a ClusterQueue can consume a + quantity of quota equal to nominalQuota+borrowingLimit, assuming the other + ClusterQueues in the cohort have enough unused quota. + If null, it means that there is no borrowing limit. + If not null, it must be non-negative. + borrowingLimit must be null if spec.cohort is empty. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + lendingLimit: + anyOf: + - type: integer + - type: string + description: |- + lendingLimit is the maximum amount of unused quota for the [flavor, resource] + combination that this ClusterQueue can lend to other ClusterQueues in the same cohort. + In total, at a given time, ClusterQueue reserves for its exclusive use + a quantity of quota equals to nominalQuota - lendingLimit. + If null, it means that there is no lending limit, meaning that + all the nominalQuota can be borrowed by other clusterQueues in the cohort. + If not null, it must be non-negative. + lendingLimit must be null if spec.cohort is empty. + This field is in beta stage and is enabled by default. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name of this resource. + type: string + nominalQuota: + anyOf: + - type: integer + - type: string + description: |- + nominalQuota is the quantity of this resource that is available for + Workloads admitted by this ClusterQueue at a point in time. + The nominalQuota must be non-negative. + nominalQuota should represent the resources in the cluster available for + running jobs (after discounting resources consumed by system components + and pods not managed by kueue). In an autoscaled cluster, nominalQuota + should account for resources that can be provided by a component such as + Kubernetes cluster-autoscaler. + + If the ClusterQueue belongs to a cohort, the sum of the quotas for each + (flavor, resource) combination defines the maximum quantity that can be + allocated by a ClusterQueue in the cohort. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + - nominalQuota + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - coveredResources + - flavors + type: object + x-kubernetes-validations: + - message: flavors must have the same number of resources as the + coveredResources + rule: self.flavors.all(x, size(x.resources) == size(self.coveredResources)) + maxItems: 16 + type: array + x-kubernetes-list-type: atomic + stopPolicy: + default: None + description: |- + stopPolicy - if set to a value different from None, the ClusterQueue is considered Inactive, no new reservation being + made. + + Depending on its value, its associated workloads will: + + - None - Workloads are admitted + - HoldAndDrain - Admitted workloads are evicted and Reserving workloads will cancel the reservation. + - Hold - Admitted workloads will run to completion and Reserving workloads will cancel the reservation. + enum: + - None + - Hold + - HoldAndDrain + type: string + type: object + x-kubernetes-validations: + - message: borrowingLimit must be nil when cohort is empty + rule: '!has(self.cohort) && has(self.resourceGroups) ? self.resourceGroups.all(rg, + rg.flavors.all(f, f.resources.all(r, !has(r.borrowingLimit)))) : true' + status: + description: ClusterQueueStatus defines the observed state of ClusterQueue + properties: + admittedWorkloads: + description: |- + admittedWorkloads is the number of workloads currently admitted to this + clusterQueue and haven't finished yet. + format: int32 + type: integer + conditions: + description: |- + conditions hold the latest available observations of the ClusterQueue + current state. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + fairSharing: + description: FairSharing contains the information about the current + status of fair sharing. + properties: + weightedShare: + description: |- + WeightedShare represent the maximum of the ratios of usage above nominal + quota to the lendable resources in the cohort, among all the resources + provided by the ClusterQueue, and divided by the weight. + If zero, it means that the usage of the ClusterQueue is below the nominal quota. + If the ClusterQueue has a weight of zero, this will return 9223372036854775807, + the maximum possible share value. + format: int64 + type: integer + required: + - weightedShare + type: object + flavorsReservation: + description: |- + flavorsReservation are the reserved quotas, by flavor, currently in use by the + workloads assigned to this ClusterQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: resources lists the quota usage for the resources + in this flavor. + items: + properties: + borrowed: + anyOf: + - type: integer + - type: string + description: |- + Borrowed is quantity of quota that is borrowed from the cohort. In other + words, it's the used quota that is over the nominalQuota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name of the resource + type: string + total: + anyOf: + - type: integer + - type: string + description: |- + total is the total quantity of used quota, including the amount borrowed + from the cohort. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + flavorsUsage: + description: |- + flavorsUsage are the used quotas, by flavor, currently in use by the + workloads admitted in this ClusterQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: resources lists the quota usage for the resources + in this flavor. + items: + properties: + borrowed: + anyOf: + - type: integer + - type: string + description: |- + Borrowed is quantity of quota that is borrowed from the cohort. In other + words, it's the used quota that is over the nominalQuota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name of the resource + type: string + total: + anyOf: + - type: integer + - type: string + description: |- + total is the total quantity of used quota, including the amount borrowed + from the cohort. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + pendingWorkloads: + description: |- + pendingWorkloads is the number of workloads currently waiting to be + admitted to this clusterQueue. + format: int32 + type: integer + pendingWorkloadsStatus: + description: |- + PendingWorkloadsStatus contains the information exposed about the current + status of the pending workloads in the cluster queue. + Deprecated: This field will be removed on v1beta2, use VisibilityOnDemand + (https://kueue.sigs.k8s.io/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand/) + instead. + properties: + clusterQueuePendingWorkload: + description: Head contains the list of top pending workloads. + items: + description: |- + ClusterQueuePendingWorkload contains the information identifying a pending workload + in the cluster queue. + properties: + name: + description: Name indicates the name of the pending workload. + type: string + namespace: + description: Namespace indicates the name of the pending + workload. + type: string + required: + - name + - namespace + type: object + type: array + x-kubernetes-list-type: atomic + lastChangeTime: + description: LastChangeTime indicates the time of the last change + of the structure. + format: date-time + type: string + required: + - lastChangeTime + type: object + reservingWorkloads: + description: |- + reservingWorkloads is the number of workloads currently reserving quota in this + clusterQueue. + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: cohorts.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: Cohort + listKind: CohortList + plural: cohorts + singular: cohort + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + Cohort is the Schema for the cohorts API. Using Hierarchical + Cohorts (any Cohort which has a parent) with Fair Sharing + results in undefined behavior in 0.9 + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CohortSpec defines the desired state of Cohort + properties: + parent: + description: |- + Parent references the name of the Cohort's parent, if + any. It satisfies one of three cases: + 1) Unset. This Cohort is the root of its Cohort tree. + 2) References a non-existent Cohort. We use default Cohort (no borrowing/lending limits). + 3) References an existent Cohort. + + If a cycle is created, we disable all members of the + Cohort, including ClusterQueues, until the cycle is + removed. We prevent further admission while the cycle + exists. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resourceGroups: + description: |- + ResourceGroups describes groupings of Resources and + Flavors. Each ResourceGroup defines a list of Resources + and a list of Flavors which provide quotas for these + Resources. Each Resource and each Flavor may only form part + of one ResourceGroup. There may be up to 16 ResourceGroups + within a Cohort. + + BorrowingLimit limits how much members of this Cohort + subtree can borrow from the parent subtree. + + LendingLimit limits how much members of this Cohort subtree + can lend to the parent subtree. + + Borrowing and Lending limits must only be set when the + Cohort has a parent. Otherwise, the Cohort create/update + will be rejected by the webhook. + items: + properties: + coveredResources: + description: |- + coveredResources is the list of resources covered by the flavors in this + group. + Examples: cpu, memory, vendor.com/gpu. + The list cannot be empty and it can contain up to 16 resources. + items: + description: ResourceName is the name identifying various + resources in a ResourceList. + type: string + maxItems: 16 + minItems: 1 + type: array + flavors: + description: |- + flavors is the list of flavors that provide the resources of this group. + Typically, different flavors represent different hardware models + (e.g., gpu models, cpu architectures) or pricing models (on-demand vs spot + cpus). + Each flavor MUST list all the resources listed for this group in the same + order as the .resources field. + The list cannot be empty and it can contain up to 16 flavors. + items: + properties: + name: + description: |- + name of this flavor. The name should match the .metadata.name of a + ResourceFlavor. If a matching ResourceFlavor does not exist, the + ClusterQueue will have an Active condition set to False. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: |- + resources is the list of quotas for this flavor per resource. + There could be up to 16 resources. + items: + properties: + borrowingLimit: + anyOf: + - type: integer + - type: string + description: |- + borrowingLimit is the maximum amount of quota for the [flavor, resource] + combination that this ClusterQueue is allowed to borrow from the unused + quota of other ClusterQueues in the same cohort. + In total, at a given time, Workloads in a ClusterQueue can consume a + quantity of quota equal to nominalQuota+borrowingLimit, assuming the other + ClusterQueues in the cohort have enough unused quota. + If null, it means that there is no borrowing limit. + If not null, it must be non-negative. + borrowingLimit must be null if spec.cohort is empty. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + lendingLimit: + anyOf: + - type: integer + - type: string + description: |- + lendingLimit is the maximum amount of unused quota for the [flavor, resource] + combination that this ClusterQueue can lend to other ClusterQueues in the same cohort. + In total, at a given time, ClusterQueue reserves for its exclusive use + a quantity of quota equals to nominalQuota - lendingLimit. + If null, it means that there is no lending limit, meaning that + all the nominalQuota can be borrowed by other clusterQueues in the cohort. + If not null, it must be non-negative. + lendingLimit must be null if spec.cohort is empty. + This field is in beta stage and is enabled by default. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name of this resource. + type: string + nominalQuota: + anyOf: + - type: integer + - type: string + description: |- + nominalQuota is the quantity of this resource that is available for + Workloads admitted by this ClusterQueue at a point in time. + The nominalQuota must be non-negative. + nominalQuota should represent the resources in the cluster available for + running jobs (after discounting resources consumed by system components + and pods not managed by kueue). In an autoscaled cluster, nominalQuota + should account for resources that can be provided by a component such as + Kubernetes cluster-autoscaler. + + If the ClusterQueue belongs to a cohort, the sum of the quotas for each + (flavor, resource) combination defines the maximum quantity that can be + allocated by a ClusterQueue in the cohort. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + - nominalQuota + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - coveredResources + - flavors + type: object + x-kubernetes-validations: + - message: flavors must have the same number of resources as the + coveredResources + rule: self.flavors.all(x, size(x.resources) == size(self.coveredResources)) + maxItems: 16 + type: array + x-kubernetes-list-type: atomic + type: object + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: localqueues.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: LocalQueue + listKind: LocalQueueList + plural: localqueues + shortNames: + - queue + - queues + - lq + singular: localqueue + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Backing ClusterQueue + jsonPath: .spec.clusterQueue + name: ClusterQueue + type: string + - description: Number of pending workloads + jsonPath: .status.pendingWorkloads + name: Pending Workloads + type: integer + - description: Number of admitted workloads that haven't finished yet. + jsonPath: .status.admittedWorkloads + name: Admitted Workloads + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + description: LocalQueue is the Schema for the localQueues API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: LocalQueueSpec defines the desired state of LocalQueue + properties: + clusterQueue: + description: clusterQueue is a reference to a clusterQueue that backs + this localQueue. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + stopPolicy: + default: None + description: |- + stopPolicy - if set to a value different from None, the LocalQueue is considered Inactive, + no new reservation being made. + + Depending on its value, its associated workloads will: + + - None - Workloads are admitted + - HoldAndDrain - Admitted workloads are evicted and Reserving workloads will cancel the reservation. + - Hold - Admitted workloads will run to completion and Reserving workloads will cancel the reservation. + enum: + - None + - Hold + - HoldAndDrain + type: string + type: object + status: + description: LocalQueueStatus defines the observed state of LocalQueue + properties: + admittedWorkloads: + description: |- + admittedWorkloads is the number of workloads in this LocalQueue + admitted to a ClusterQueue and that haven't finished yet. + format: int32 + type: integer + conditions: + description: |- + Conditions hold the latest available observations of the LocalQueue + current state. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + flavorUsage: + description: |- + flavorsUsage are the used quotas, by flavor currently in use by the + workloads assigned to this LocalQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: resources lists the quota usage for the resources + in this flavor. + items: + properties: + name: + description: name of the resource. + type: string + total: + anyOf: + - type: integer + - type: string + description: total is the total quantity of used quota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + flavors: + description: flavors lists all currently available ResourceFlavors + in specified ClusterQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + nodeLabels: + additionalProperties: + type: string + description: |- + nodeLabels are labels that associate the ResourceFlavor with Nodes that + have the same labels. + maxProperties: 8 + type: object + x-kubernetes-map-type: atomic + nodeTaints: + description: |- + nodeTaints are taints that the nodes associated with this ResourceFlavor + have. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to + a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint + key. + type: string + required: + - effect + - key + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + resources: + description: resources used in the flavor. + items: + description: ResourceName is the name identifying various + resources in a ResourceList. + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + flavorsReservation: + description: |- + flavorsReservation are the reserved quotas, by flavor currently in use by the + workloads assigned to this LocalQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: resources lists the quota usage for the resources + in this flavor. + items: + properties: + name: + description: name of the resource. + type: string + total: + anyOf: + - type: integer + - type: string + description: total is the total quantity of used quota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + pendingWorkloads: + description: PendingWorkloads is the number of Workloads in the LocalQueue + not yet admitted to a ClusterQueue + format: int32 + type: integer + reservingWorkloads: + description: |- + reservingWorkloads is the number of workloads in this LocalQueue + reserving quota in a ClusterQueue and that haven't finished yet. + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: multikueueclusters.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: MultiKueueCluster + listKind: MultiKueueClusterList + plural: multikueueclusters + singular: multikueuecluster + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: MultiKueueCluster is the Schema for the multikueue API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + kubeConfig: + description: Information how to connect to the cluster. + properties: + location: + description: |- + Location of the KubeConfig. + + If LocationType is Secret then Location is the name of the secret inside the namespace in + which the kueue controller manager is running. The config should be stored in the "kubeconfig" key. + type: string + locationType: + default: Secret + description: Type of the KubeConfig location. + enum: + - Secret + - Path + type: string + required: + - location + - locationType + type: object + required: + - kubeConfig + type: object + status: + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: multikueueconfigs.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: MultiKueueConfig + listKind: MultiKueueConfigList + plural: multikueueconfigs + singular: multikueueconfig + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: MultiKueueConfig is the Schema for the multikueue API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: MultiKueueConfigSpec defines the desired state of MultiKueueConfig + properties: + clusters: + description: List of MultiKueueClusters names where the workloads + from the ClusterQueue should be distributed. + items: + type: string + maxItems: 10 + minItems: 1 + type: array + x-kubernetes-list-type: set + required: + - clusters + type: object + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: provisioningrequestconfigs.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: ProvisioningRequestConfig + listKind: ProvisioningRequestConfigList + plural: provisioningrequestconfigs + singular: provisioningrequestconfig + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: ProvisioningRequestConfig is the Schema for the provisioningrequestconfig + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ProvisioningRequestConfigSpec defines the desired state of + ProvisioningRequestConfig + properties: + managedResources: + description: |- + managedResources contains the list of resources managed by the autoscaling. + + If empty, all resources are considered managed. + + If not empty, the ProvisioningRequest will contain only the podsets that are + requesting at least one of them. + + If none of the workloads podsets is requesting at least a managed resource, + the workload is considered ready. + items: + description: ResourceName is the name identifying various resources + in a ResourceList. + type: string + maxItems: 100 + type: array + x-kubernetes-list-type: set + parameters: + additionalProperties: + description: Parameter is limited to 255 characters. + maxLength: 255 + type: string + description: Parameters contains all other parameters classes may + require. + maxProperties: 100 + type: object + provisioningClassName: + description: |- + ProvisioningClassName describes the different modes of provisioning the resources. + Check autoscaling.x-k8s.io ProvisioningRequestSpec.ProvisioningClassName for details. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + retryStrategy: + default: + backoffBaseSeconds: 60 + backoffLimitCount: 3 + backoffMaxSeconds: 1800 + description: |- + retryStrategy defines strategy for retrying ProvisioningRequest. + If null, then the default configuration is applied with the following parameter values: + backoffLimitCount: 3 + backoffBaseSeconds: 60 - 1 min + backoffMaxSeconds: 1800 - 30 mins + + To switch off retry mechanism + set retryStrategy.backoffLimitCount to 0. + properties: + backoffBaseSeconds: + default: 60 + description: |- + BackoffBaseSeconds defines the base for the exponential backoff for + re-queuing an evicted workload. + + Defaults to 60. + format: int32 + type: integer + backoffLimitCount: + default: 3 + description: |- + BackoffLimitCount defines the maximum number of re-queuing retries. + Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). + + Every backoff duration is about "b*2^(n-1)+Rand" where: + - "b" represents the base set by "BackoffBaseSeconds" parameter, + - "n" represents the "workloadStatus.requeueState.count", + - "Rand" represents the random jitter. + During this time, the workload is taken as an inadmissible and + other workloads will have a chance to be admitted. + By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...). + + Defaults to 3. + format: int32 + type: integer + backoffMaxSeconds: + default: 1800 + description: |- + BackoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload. + + Defaults to 1800. + format: int32 + type: integer + type: object + required: + - provisioningClassName + type: object + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: resourceflavors.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: ResourceFlavor + listKind: ResourceFlavorList + plural: resourceflavors + shortNames: + - flavor + - flavors + - rf + singular: resourceflavor + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: ResourceFlavor is the Schema for the resourceflavors API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ResourceFlavorSpec defines the desired state of the ResourceFlavor + properties: + nodeLabels: + additionalProperties: + type: string + description: |- + nodeLabels are labels that associate the ResourceFlavor with Nodes that + have the same labels. + When a Workload is admitted, its podsets can only get assigned + ResourceFlavors whose nodeLabels match the nodeSelector and nodeAffinity + fields. + Once a ResourceFlavor is assigned to a podSet, the ResourceFlavor's + nodeLabels should be injected into the pods of the Workload by the + controller that integrates with the Workload object. + + nodeLabels can be up to 8 elements. + maxProperties: 8 + type: object + x-kubernetes-map-type: atomic + nodeTaints: + description: |- + nodeTaints are taints that the nodes associated with this ResourceFlavor + have. + Workloads' podsets must have tolerations for these nodeTaints in order to + get assigned this ResourceFlavor during admission. + + An example of a nodeTaint is + cloud.provider.com/preemptible="true":NoSchedule + + nodeTaints can be up to 8 elements. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: 'supported taint effect values: ''NoSchedule'', ''PreferNoSchedule'', + ''NoExecute''' + rule: self.all(x, x.effect in ['NoSchedule', 'PreferNoSchedule', + 'NoExecute']) + tolerations: + description: |- + tolerations are extra tolerations that will be added to the pods admitted in + the quota associated with this resource flavor. + + An example of a toleration is + cloud.provider.com/preemptible="true":NoSchedule + + tolerations can be up to 8 elements. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: operator must be Exists when 'key' is empty, which means + 'match all values and all keys' + rule: 'self.all(x, !has(x.key) ? x.operator == ''Exists'' : true)' + - message: effect must be 'NoExecute' when 'tolerationSeconds' is + set + rule: 'self.all(x, has(x.tolerationSeconds) ? x.effect == ''NoExecute'' + : true)' + - message: 'supported toleration values: ''Equal''(default), ''Exists''' + rule: self.all(x, !has(x.operator) || x.operator in ['Equal', 'Exists']) + - message: a value must be empty when 'operator' is 'Exists' + rule: 'self.all(x, has(x.operator) && x.operator == ''Exists'' ? + !has(x.value) : true)' + - message: 'supported taint effect values: ''NoSchedule'', ''PreferNoSchedule'', + ''NoExecute''' + rule: self.all(x, !has(x.effect) || x.effect in ['NoSchedule', 'PreferNoSchedule', + 'NoExecute']) + topologyName: + description: |- + topologyName indicates topology for the TAS ResourceFlavor. + When specified, it enables scraping of the topology information from the + nodes matching to the Resource Flavor node labels. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + type: object + x-kubernetes-validations: + - message: at least one nodeLabel is required when topology is set + rule: '!has(self.topologyName) || self.nodeLabels.size() >= 1' + - message: resourceFlavorSpec are immutable when topologyName is set + rule: '!has(oldSelf.topologyName) || self == oldSelf' + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: topologies.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: Topology + listKind: TopologyList + plural: topologies + singular: topology + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Topology is the Schema for the topology API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: TopologySpec defines the desired state of Topology + properties: + levels: + description: levels define the levels of topology. + items: + description: TopologyLevel defines the desired state of TopologyLevel + properties: + nodeLabel: + description: |- + nodeLabel indicates the name of the node label for a specific topology + level. + + Examples: + - cloud.provider.com/topology-block + - cloud.provider.com/topology-rack + maxLength: 316 + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - nodeLabel + type: object + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + - message: must be unique + rule: size(self.filter(i, size(self.filter(j, j == i)) > 1)) == + 0 + - message: the kubernetes.io/hostname label can only be used at the + lowest level of topology + rule: size(self.filter(i, i.nodeLabel == 'kubernetes.io/hostname')) + == 0 || self[size(self) - 1].nodeLabel == 'kubernetes.io/hostname' + required: + - levels + type: object + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: workloadpriorityclasses.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: WorkloadPriorityClass + listKind: WorkloadPriorityClassList + plural: workloadpriorityclasses + singular: workloadpriorityclass + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Value of workloadPriorityClass's Priority + jsonPath: .value + name: Value + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + description: WorkloadPriorityClass is the Schema for the workloadPriorityClass + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + description: + description: |- + description is an arbitrary string that usually provides guidelines on + when this workloadPriorityClass should be used. + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + value: + description: |- + value represents the integer value of this workloadPriorityClass. This is the actual priority that workloads + receive when jobs have the name of this class in their workloadPriorityClass label. + Changing the value of workloadPriorityClass doesn't affect the priority of workloads that were already created. + format: int32 + type: integer + required: + - value + type: object + served: true + storage: true + subresources: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: workloads.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: Workload + listKind: WorkloadList + plural: workloads + shortNames: + - wl + singular: workload + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Name of the queue this workload was submitted to + jsonPath: .spec.queueName + name: Queue + type: string + - description: Name of the ClusterQueue where the workload is reserving quota + jsonPath: .status.admission.clusterQueue + name: Reserved in + type: string + - description: Admission status + jsonPath: .status.conditions[?(@.type=='Admitted')].status + name: Admitted + type: string + - description: Workload finished + jsonPath: .status.conditions[?(@.type=='Finished')].status + name: Finished + type: string + - description: Time this workload was created + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Workload is the Schema for the workloads API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: WorkloadSpec defines the desired state of Workload + properties: + active: + default: true + description: |- + Active determines if a workload can be admitted into a queue. + Changing active from true to false will evict any running workloads. + Possible values are: + + - false: indicates that a workload should never be admitted and evicts running workloads + - true: indicates that a workload can be evaluated for admission into it's respective queue. + + Defaults to true + type: boolean + maximumExecutionTimeSeconds: + description: |- + maximumExecutionTimeSeconds if provided, determines the maximum time, in seconds, + the workload can be admitted before it's automatically deactivated. + + If unspecified, no execution time limit is enforced on the Workload. + format: int32 + minimum: 1 + type: integer + podSets: + description: |- + podSets is a list of sets of homogeneous pods, each described by a Pod spec + and a count. + There must be at least one element and at most 8. + podSets cannot be changed. + items: + properties: + count: + default: 1 + description: count is the number of pods for the spec. + format: int32 + minimum: 0 + type: integer + minCount: + description: |- + minCount is the minimum number of pods for the spec acceptable + if the workload supports partial admission. + + If not provided, partial admission for the current PodSet is not + enabled. + + Only one podSet within the workload can use this. + + This is an alpha field and requires enabling PartialAdmission feature gate. + format: int32 + minimum: 1 + type: integer + name: + default: main + description: name is the PodSet name. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + template: + description: |- + template is the Pod template. + + The only allowed fields in template.metadata are labels and annotations. + + If requests are omitted for a container or initContainer, + they default to the limits if they are explicitly specified for the + container or initContainer. + + During admission, the rules in nodeSelector and + nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution that match + the keys in the nodeLabels from the ResourceFlavors considered for this + Workload are used to filter the ResourceFlavors that can be assigned to + this podSet. + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling + rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, + in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules + (e.g. co-locate this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the same + node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates + whether a service account token should be automatically + mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container that you + want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if value + is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name of + a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of + a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS resolver + options of a pod. + properties: + name: + description: Required. + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if value + is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed for ephemeral + containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral + containers. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + This may only be set for init containers. You cannot set this field on + ephemeral containers. + type: string + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name of + a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of + a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + If this option is set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + of that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container that you + want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if value + is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name of + a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of + a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the reference + to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition + in the pod's condition list with matching type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated to a + Pod to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that + applies to the container. + type: string + role: + description: Role is a SELinux role label that + applies to the container. + type: string + type: + description: Type is a SELinux type label that + applies to the container. + type: string + user: + description: User is a SELinux user label that + applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter + to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies how + to spread matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume in a + pod that may be accessed by any container in the + pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data + Disk mount on the host and bind mount to the + pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching + mode: None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data + disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk + in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: + multiple blob disks per storage account Dedicated: + single blob disk per storage account Managed: + azure managed data disk (only in managed + availability set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File + Service mount on the host and bind mount to + the pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret + that contains Azure Storage Account Name + and Key + type: string + shareName: + description: shareName is the azure share + Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount + on the host that shares a pod's lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as the + mounted root, rather than the full Ceph + tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap + that should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the + ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) + represents ephemeral storage that is handled + by certain external CSI drivers (Beta feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API + about the pod that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API + volume file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file to + be created. Must not be absolute or + contain the ''..'' path. Must be utf-8 + encoded. The first item of the relative + path must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query + over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding + reference to the PersistentVolume + backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource + that is attached to a kubelet's host machine + and then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target lun + number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target + worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver + to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field + holds extra command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume + attached to a kubelet's host machine. This depends + on the Flocker control service being running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the + dataset. This is unique identifier of a + Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for + the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether + support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether + support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified + Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun + number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret + for iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a + PhotonController persistent disk attached and + mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx + volume attached and mounted on kubelets host + machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies + a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources + secrets, configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from + the volume root to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about + the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key + to a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether + the ConfigMap or its keys must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information + about the downwardAPI data to project + properties: + items: + description: Items is a list of + DownwardAPIVolume file + items: + description: DownwardAPIVolumeFile + represents information to create + the file containing the pod + field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only + annotations, labels, name, + namespace and uid are supported.' + properties: + apiVersion: + description: Version of + the schema the FieldPath + is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the + field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path + is the relative path name + of the file to be created. + Must not be absolute or + contain the ''..'' path. + Must be utf-8 encoded. The + first item of the relative + path must not start with + ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies + the output format of + the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information about + the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key + to a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify + whether the Secret or its key + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is + information about the serviceAccountToken + data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: quobyte represents a Quobyte mount + on the host that shares a pod's lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references + an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent + volume attached and mounted on Kubernetes nodes. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of + the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name + of the ScaleIO Protection Domain for the + configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default + false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage + Pool associated with the protection domain. + type: string + system: + description: system is the name of the storage + system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether + the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS + volume attached and mounted on Kubernetes nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere + volume attached and mounted on kubelets host + machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage + Policy Based Management (SPBM) profile ID + associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage + Policy Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies + vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object + topologyRequest: + description: topologyRequest defines the topology request for + the PodSet. + properties: + podIndexLabel: + description: |- + PodIndexLabel indicates the name of the label indexing the pods. + For example, in the context of + - kubernetes job this is: kubernetes.io/job-completion-index + - JobSet: kubernetes.io/job-completion-index (inherited from Job) + - Kubeflow: training.kubeflow.org/replica-index + type: string + preferred: + description: |- + preferred indicates the topology level preferred by the PodSet, as + indicated by the `kueue.x-k8s.io/podset-preferred-topology` PodSet + annotation. + type: string + required: + description: |- + required indicates the topology level required by the PodSet, as + indicated by the `kueue.x-k8s.io/podset-required-topology` PodSet + annotation. + type: string + subGroupCount: + description: |- + SubGroupIndexLabel indicates the count of replicated Jobs (groups) within a PodSet. + For example, in the context of JobSet this value is read from jobset.sigs.k8s.io/replicatedjob-replicas. + format: int32 + type: integer + subGroupIndexLabel: + description: |- + SubGroupIndexLabel indicates the name of the label indexing the instances of replicated Jobs (groups) + within a PodSet. For example, in the context of JobSet this is jobset.sigs.k8s.io/job-index. + type: string + type: object + required: + - count + - template + type: object + x-kubernetes-validations: + - message: minCount should be positive and less or equal to count + rule: 'has(self.minCount) ? self.minCount <= self.count : true' + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + priority: + description: |- + Priority determines the order of access to the resources managed by the + ClusterQueue where the workload is queued. + The priority value is populated from PriorityClassName. + The higher the value, the higher the priority. + If priorityClassName is specified, priority must not be null. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the workload's priority. + "system-node-critical" and "system-cluster-critical" are two special + keywords which indicate the highest priorities with the former being + the highest priority. Any other name must be defined by creating a + PriorityClass object with that name. If not specified, the workload + priority will be default or zero if there is no default. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + priorityClassSource: + default: "" + description: |- + priorityClassSource determines whether the priorityClass field refers to a pod PriorityClass or kueue.x-k8s.io/workloadpriorityclass. + Workload's PriorityClass can accept the name of a pod priorityClass or a workloadPriorityClass. + When using pod PriorityClass, a priorityClassSource field has the scheduling.k8s.io/priorityclass value. + enum: + - kueue.x-k8s.io/workloadpriorityclass + - scheduling.k8s.io/priorityclass + - "" + type: string + queueName: + description: |- + queueName is the name of the LocalQueue the Workload is associated with. + queueName cannot be changed while .status.admission is not null. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - podSets + type: object + x-kubernetes-validations: + - message: priority should not be nil when priorityClassName is set + rule: 'has(self.priorityClassName) ? has(self.priority) : true' + status: + description: WorkloadStatus defines the observed state of Workload + properties: + accumulatedPastExexcutionTimeSeconds: + description: |- + accumulatedPastExexcutionTimeSeconds holds the total time, in seconds, the workload spent + in Admitted state, in the previous `Admit` - `Evict` cycles. + format: int32 + type: integer + admission: + description: |- + admission holds the parameters of the admission of the workload by a + ClusterQueue. admission can be set back to null, but its fields cannot be + changed once set. + properties: + clusterQueue: + description: clusterQueue is the name of the ClusterQueue that + admitted this workload. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + podSetAssignments: + description: PodSetAssignments hold the admission results for + each of the .spec.podSets entries. + items: + properties: + count: + description: |- + count is the number of pods taken into account at admission time. + This field will not change in case of quota reclaim. + Value could be missing for Workloads created before this field was added, + in that case spec.podSets[*].count value will be used. + format: int32 + minimum: 0 + type: integer + flavors: + additionalProperties: + description: ResourceFlavorReference is the name of the + ResourceFlavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + description: Flavors are the flavors assigned to the workload + for each resource. + type: object + name: + default: main + description: Name is the name of the podSet. It should match + one of the names in .spec.podSets. + maxLength: 63 + pattern: ^(?i)[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + resourceUsage: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + resourceUsage keeps track of the total resources all the pods in the podset need to run. + + Beside what is provided in podSet's specs, this calculation takes into account + the LimitRange defaults and RuntimeClass overheads at the moment of admission. + This field will not change in case of quota reclaim. + type: object + topologyAssignment: + description: |- + topologyAssignment indicates the topology assignment divided into + topology domains corresponding to the lowest level of the topology. + The assignment specifies the number of Pods to be scheduled per topology + domain and specifies the node selectors for each topology domain, in the + following way: the node selector keys are specified by the levels field + (same for all domains), and the corresponding node selector value is + specified by the domains.values subfield. If the TopologySpec.Levels field contains + "kubernetes.io/hostname" label, topologyAssignment will contain data only for + this label, and omit higher levels in the topology + + Example: + + topologyAssignment: + levels: + - cloud.provider.com/topology-block + - cloud.provider.com/topology-rack + domains: + - values: [block-1, rack-1] + count: 4 + - values: [block-1, rack-2] + count: 2 + + Here: + - 4 Pods are to be scheduled on nodes matching the node selector: + cloud.provider.com/topology-block: block-1 + cloud.provider.com/topology-rack: rack-1 + - 2 Pods are to be scheduled on nodes matching the node selector: + cloud.provider.com/topology-block: block-1 + cloud.provider.com/topology-rack: rack-2 + + Example: + Below there is an equivalent of the above example assuming, Topology + object defines kubernetes.io/hostname as the lowest level in topology. + Hence we omit higher level of topologies, since the hostname label + is sufficient to explicitly identify a proper node. + + topologyAssignment: + levels: + - kubernetes.io/hostname + domains: + - values: [hostname-1] + count: 4 + - values: [hostname-2] + count: 2 + properties: + domains: + description: |- + domains is a list of topology assignments split by topology domains at + the lowest level of the topology. + items: + properties: + count: + description: |- + count indicates the number of Pods to be scheduled in the topology + domain indicated by the values field. + format: int32 + minimum: 1 + type: integer + values: + description: |- + values is an ordered list of node selector values describing a topology + domain. The values correspond to the consecutive topology levels, from + the highest to the lowest. + items: + type: string + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + required: + - count + - values + type: object + type: array + levels: + description: |- + levels is an ordered list of keys denoting the levels of the assigned + topology (i.e. node label keys), from the highest to the lowest level of + the topology. + items: + type: string + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + required: + - domains + - levels + type: object + required: + - name + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - clusterQueue + - podSetAssignments + type: object + admissionChecks: + description: admissionChecks list all the admission checks required + by the workload and the current status + items: + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + name: + description: name identifies the admission check. + maxLength: 316 + type: string + podSetUpdates: + items: + description: |- + PodSetUpdate contains a list of pod set modifications suggested by AdmissionChecks. + The modifications should be additive only - modifications of already existing keys + or having the same key provided by multiple AdmissionChecks is not allowed and will + result in failure during workload admission. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + name: + description: Name of the PodSet to modify. Should match + to one of the Workload's PodSets. + type: string + nodeSelector: + additionalProperties: + type: string + type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + maxItems: 8 + type: array + x-kubernetes-validations: + - message: operator must be Exists when 'key' is empty, + which means 'match all values and all keys' + rule: 'self.all(x, !has(x.key) ? x.operator == ''Exists'' + : true)' + - message: effect must be 'NoExecute' when 'tolerationSeconds' + is set + rule: 'self.all(x, has(x.tolerationSeconds) ? x.effect + == ''NoExecute'' : true)' + - message: 'supported toleration values: ''Equal''(default), + ''Exists''' + rule: self.all(x, !has(x.operator) || x.operator in + ['Equal', 'Exists']) + - message: a value must be empty when 'operator' is 'Exists' + rule: 'self.all(x, has(x.operator) && x.operator == + ''Exists'' ? !has(x.value) : true)' + - message: 'supported taint effect values: ''NoSchedule'', + ''PreferNoSchedule'', ''NoExecute''' + rule: self.all(x, !has(x.effect) || x.effect in ['NoSchedule', + 'PreferNoSchedule', 'NoExecute']) + required: + - name + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + state: + description: state of the admissionCheck, one of Pending, Ready, + Retry, Rejected + enum: + - Pending + - Ready + - Retry + - Rejected + type: string + required: + - lastTransitionTime + - message + - name + - state + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + conditions: + description: |- + conditions hold the latest available observations of the Workload + current state. + + The type of the condition could be: + + - Admitted: the Workload was admitted through a ClusterQueue. + - Finished: the associated workload finished running (failed or succeeded). + - PodsReady: at least `.spec.podSets[*].count` Pods are ready or have + succeeded. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + reclaimablePods: + description: |- + reclaimablePods keeps track of the number pods within a podset for which + the resource reservation is no longer needed. + items: + properties: + count: + description: count is the number of pods for which the requested + resources are no longer needed. + format: int32 + minimum: 0 + type: integer + name: + description: name is the PodSet name. + type: string + required: + - count + - name + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + requeueState: + description: |- + requeueState holds the re-queue state + when a workload meets Eviction with PodsReadyTimeout reason. + properties: + count: + description: |- + count records the number of times a workload has been re-queued + When a deactivated (`.spec.activate`=`false`) workload is reactivated (`.spec.activate`=`true`), + this count would be reset to null. + format: int32 + minimum: 0 + type: integer + requeueAt: + description: |- + requeueAt records the time when a workload will be re-queued. + When a deactivated (`.spec.activate`=`false`) workload is reactivated (`.spec.activate`=`true`), + this time would be reset to null. + format: date-time + type: string + type: object + resourceRequests: + description: |- + resourceRequests provides a detailed view of the resources that were + requested by a non-admitted workload when it was considered for admission. + If admission is non-null, resourceRequests will be empty because + admission.resourceUsage contains the detailed information. + items: + properties: + name: + default: main + description: name is the name of the podSet. It should match + one of the names in .spec.podSets. + maxLength: 63 + pattern: ^(?i)[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + resources is the total resources all the pods in the podset need to run. + + Beside what is provided in podSet's specs, this value also takes into account + the LimitRange defaults and RuntimeClass overheads at the moment of consideration + and the application of resource.excludeResourcePrefixes and resource.transformations. + type: object + required: + - name + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + type: object + x-kubernetes-validations: + - message: podSetAssignments must have the same number of podSets as the spec + rule: 'has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, + c.type == ''QuotaReserved'' && c.status == ''True'') && has(self.status.admission) + ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) + : true' + - message: field is immutable + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, + c.type == ''QuotaReserved'' && c.status == ''True'')) ? (oldSelf.spec.priorityClassSource + == self.spec.priorityClassSource) : true' + - message: field is immutable + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, + c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassName) + && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName + == self.spec.priorityClassName) : true' + - message: field is immutable + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, + c.type == ''QuotaReserved'' && c.status == ''True'')) && (has(self.status) + && has(self.status.conditions) && self.status.conditions.exists(c, c.type + == ''QuotaReserved'' && c.status == ''True'')) && has(oldSelf.spec.queueName) + && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName + : true' + - message: maximumExecutionTimeSeconds is immutable while admitted + rule: ((has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, + c.type == 'Admitted' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) + && self.status.conditions.exists(c, c.type == 'Admitted' && c.status == + 'True')))?((has(oldSelf.spec.maximumExecutionTimeSeconds)?oldSelf.spec.maximumExecutionTimeSeconds:0) + == (has(self.spec.maximumExecutionTimeSeconds)?self.spec.maximumExecutionTimeSeconds:0)):true + served: true + storage: true + subresources: + status: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-controller-manager + namespace: kueue-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-leader-election-role + namespace: kueue-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.kueue.x-k8s.io/batch-admin: "true" +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-batch-admin-role +--- +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.kueue.x-k8s.io/batch-user: "true" +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-batch-user-role +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-clusterqueue-editor-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-clusterqueue-viewer-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues + verbs: + - get + - list + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-job-editor-role +rules: +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch + resources: + - jobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-job-viewer-role +rules: +- apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch +- apiGroups: + - batch + resources: + - jobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-jobset-editor-role +rules: +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-jobset-viewer-role +rules: +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets + verbs: + - get + - list + - watch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-localqueue-editor-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - localqueues + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - localqueues/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-localqueue-viewer-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - localqueues + verbs: + - get + - list + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - localqueues/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-manager-role +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - watch +- apiGroups: + - "" + resources: + - limitranges + - namespaces + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods/finalizers + verbs: + - get + - update +- apiGroups: + - "" + resources: + - pods/status + verbs: + - get + - patch +- apiGroups: + - "" + resources: + - podtemplates + verbs: + - create + - delete + - get + - list + - update + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - get + - list + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingadmissionpolicies + - validatingadmissionpolicybindings + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - replicasets + - statefulsets + verbs: + - get + - list + - watch +- apiGroups: + - autoscaling.x-k8s.io + resources: + - provisioningrequests + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - autoscaling.x-k8s.io + resources: + - provisioningrequests/status + verbs: + - get +- apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - batch + resources: + - jobs/finalizers + - jobs/status + verbs: + - get + - patch + - update +- apiGroups: + - flowcontrol.apiserver.k8s.io + resources: + - flowschemas + - prioritylevelconfigurations + verbs: + - list + - watch +- apiGroups: + - flowcontrol.apiserver.k8s.io + resources: + - flowschemas/status + verbs: + - patch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/finalizers + verbs: + - get + - update +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/status + verbs: + - get + - patch + - update +- apiGroups: + - kubeflow.org + resources: + - mpijobs + - mxjobs + - paddlejobs + - pytorchjobs + - tfjobs + - xgboostjobs + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - mpijobs/finalizers + - mxjobs/finalizers + - mxjobs/status + - paddlejobs/finalizers + - pytorchjobs/finalizers + - tfjobs/finalizers + - xgboostjobs/finalizers + verbs: + - get + - update +- apiGroups: + - kubeflow.org + resources: + - mpijobs/status + - paddlejobs/status + - pytorchjobs/status + - tfjobs/status + - xgboostjobs/status + verbs: + - get + - patch + - update +- apiGroups: + - kueue.x-k8s.io + resources: + - admissionchecks + - clusterqueues + - cohorts + - localqueues + - workloads + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - admissionchecks/finalizers + - clusterqueues/finalizers + - localqueues/finalizers + - resourceflavors/finalizers + - topology/finalizers + - workloads/finalizers + verbs: + - update +- apiGroups: + - kueue.x-k8s.io + resources: + - admissionchecks/status + - clusterqueues/status + - localqueues/status + - multikueueclusters/status + - workloads/status + verbs: + - get + - patch + - update +- apiGroups: + - kueue.x-k8s.io + resources: + - multikueueclusters + - multikueueconfigs + - provisioningrequestconfigs + - topologies + - workloadpriorityclasses + verbs: + - get + - list + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - resourceflavors + verbs: + - delete + - get + - list + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - topology + verbs: + - get + - list + - update + - watch +- apiGroups: + - node.k8s.io + resources: + - runtimeclasses + verbs: + - get + - list + - watch +- apiGroups: + - ray.io + resources: + - rayclusters + - rayjobs + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - ray.io + resources: + - rayclusters/finalizers + - rayclusters/status + - rayjobs/finalizers + - rayjobs/status + verbs: + - get + - update +- apiGroups: + - scheduling.k8s.io + resources: + - priorityclasses + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-mpijob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - mpijobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - mpijobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-mpijob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - mpijobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - mpijobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-mxjob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - mxjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - mxjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-mxjob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - mxjobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - mxjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-paddlejob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - paddlejobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - paddlejobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-paddlejob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - paddlejobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - paddlejobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-pending-workloads-cq-viewer-role +rules: +- apiGroups: + - visibility.kueue.x-k8s.io + resources: + - clusterqueues/pendingworkloads + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-pending-workloads-lq-viewer-role +rules: +- apiGroups: + - visibility.kueue.x-k8s.io + resources: + - localqueues/pendingworkloads + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-pytorchjob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-pytorchjob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-raycluster-editor-role +rules: +- apiGroups: + - ray.io + resources: + - rayclusters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ray.io + resources: + - rayclusters/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-raycluster-viewer-role +rules: +- apiGroups: + - ray.io + resources: + - rayclusters + verbs: + - get + - list + - watch +- apiGroups: + - ray.io + resources: + - rayclusters/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-rayjob-editor-role +rules: +- apiGroups: + - ray.io + resources: + - rayjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ray.io + resources: + - rayjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-rayjob-viewer-role +rules: +- apiGroups: + - ray.io + resources: + - rayjobs + verbs: + - get + - list + - watch +- apiGroups: + - ray.io + resources: + - rayjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-resourceflavor-editor-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - resourceflavors + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-resourceflavor-viewer-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - resourceflavors + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-tfjob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - tfjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - tfjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-tfjob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - tfjobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - tfjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-workload-editor-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - workloads + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - workloads/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-workload-viewer-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - workloads + verbs: + - get + - list + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - workloads/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-xgboostjob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - xgboostjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - xgboostjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-xgboostjob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - xgboostjobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - xgboostjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-visibility-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: kueue-controller-manager + namespace: kueue-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-leader-election-rolebinding + namespace: kueue-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kueue-leader-election-role +subjects: +- kind: ServiceAccount + name: kueue-controller-manager + namespace: kueue-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kueue-manager-role +subjects: +- kind: ServiceAccount + name: kueue-controller-manager + namespace: kueue-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kueue-proxy-role +subjects: +- kind: ServiceAccount + name: kueue-controller-manager + namespace: kueue-system +--- +apiVersion: v1 +data: + controller_manager_config.yaml: | + apiVersion: config.kueue.x-k8s.io/v1beta1 + kind: Configuration + health: + healthProbeBindAddress: :8081 + metrics: + bindAddress: :8080 + # enableClusterQueueResources: true + webhook: + port: 9443 + leaderElection: + leaderElect: true + resourceName: c1f6bfd2.kueue.x-k8s.io + controller: + groupKindConcurrency: + Job.batch: 5 + Pod: 5 + Workload.kueue.x-k8s.io: 5 + LocalQueue.kueue.x-k8s.io: 1 + Cohort.kueue.x-k8s.io: 1 + ClusterQueue.kueue.x-k8s.io: 1 + ResourceFlavor.kueue.x-k8s.io: 1 + clientConnection: + qps: 50 + burst: 100 + #pprofBindAddress: :8083 + #waitForPodsReady: + # enable: false + # timeout: 5m + # blockAdmission: false + # requeuingStrategy: + # timestamp: Eviction + # backoffLimitCount: null # null indicates infinite requeuing + # backoffBaseSeconds: 60 + # backoffMaxSeconds: 3600 + #manageJobsWithoutQueueName: true + #managedJobsNamespaceSelector: + # matchLabels: + # kueue-managed: "true" + #internalCertManagement: + # enable: false + # webhookServiceName: "" + # webhookSecretName: "" + integrations: + frameworks: + - "batch/job" + - "kubeflow.org/mpijob" + - "ray.io/rayjob" + - "ray.io/raycluster" + - "jobset.x-k8s.io/jobset" + - "kubeflow.org/mxjob" + - "kubeflow.org/paddlejob" + - "kubeflow.org/pytorchjob" + - "kubeflow.org/tfjob" + - "kubeflow.org/xgboostjob" + # - "pod" + # - "deployment" # requires enabling pod integration + # - "statefulset" # requires enabling pod integration + # externalFrameworks: + # - "Foo.v1.example.com" + # podOptions: + # namespaceSelector: + # matchExpressions: + # - key: kubernetes.io/metadata.name + # operator: NotIn + # values: [ kube-system, kueue-system ] + #fairSharing: + # enable: true + # preemptionStrategies: [LessThanOrEqualToFinalShare, LessThanInitialShare] + #resources: + # excludeResourcePrefixes: [] + # transformations: + # - input: nvidia.com/mig-4g.5gb + # strategy: Replace | Retain + # outputs: + # example.com/accelerator-memory: 5Gi + # example.com/accelerator-gpc: 4 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-manager-config + namespace: kueue-system +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-webhook-server-cert + namespace: kueue-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-controller-manager-metrics-service + namespace: kueue-system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + control-plane: controller-manager +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-visibility-server + namespace: kueue-system +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: 8082 + selector: + control-plane: controller-manager +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-webhook-service + namespace: kueue-system +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-controller-manager + namespace: kueue-system +spec: + replicas: 1 + selector: + matchLabels: + control-plane: controller-manager + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + spec: + containers: + - args: + - --config=/controller_manager_config.yaml + - --zap-log-level=2 + - --feature-gates=TopologyAwareScheduling=true + command: + - /manager + image: registry.k8s.io/kueue/kueue:v0.10.0 + imagePullPolicy: Always + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: manager + ports: + - containerPort: 8082 + name: visibility + protocol: TCP + - containerPort: 9443 + name: webhook-server + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + - mountPath: /controller_manager_config.yaml + name: manager-config + subPath: controller_manager_config.yaml + - args: + - --secure-listen-address=0.0.0.0:8443 + - --upstream=http://127.0.0.1:8080/ + - --logtostderr=true + - --v=10 + image: registry.k8s.io/kubebuilder/kube-rbac-proxy:v0.16.0 + name: kube-rbac-proxy + ports: + - containerPort: 8443 + name: https + protocol: TCP + securityContext: + runAsNonRoot: true + serviceAccountName: kueue-controller-manager + terminationGracePeriodSeconds: 10 + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: kueue-webhook-server-cert + - configMap: + name: kueue-manager-config + name: manager-config + tolerations: + - effect: NoSchedule + key: components.gke.io/gke-managed-components + operator: Equal + value: "true" +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: v1beta1.visibility.kueue.x-k8s.io +spec: + group: visibility.kueue.x-k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: kueue-visibility-server + namespace: kueue-system + version: v1beta1 + versionPriority: 100 +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate--v1-pod + failurePolicy: Fail + name: mpod.kb.io + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kueue-system + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-apps-v1-deployment + failurePolicy: Fail + name: mdeployment.kb.io + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kueue-system + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - deployments + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-batch-v1-job + failurePolicy: Fail + name: mjob.kb.io + rules: + - apiGroups: + - batch + apiVersions: + - v1 + operations: + - CREATE + resources: + - jobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-jobset-x-k8s-io-v1alpha2-jobset + failurePolicy: Fail + name: mjobset.kb.io + rules: + - apiGroups: + - jobset.x-k8s.io + apiVersions: + - v1alpha2 + operations: + - CREATE + resources: + - jobsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v1-mxjob + failurePolicy: Fail + name: mmxjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + resources: + - mxjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v1-paddlejob + failurePolicy: Fail + name: mpaddlejob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + resources: + - paddlejobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v1-pytorchjob + failurePolicy: Fail + name: mpytorchjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + resources: + - pytorchjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v1-tfjob + failurePolicy: Fail + name: mtfjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + resources: + - tfjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v1-xgboostjob + failurePolicy: Fail + name: mxgboostjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + resources: + - xgboostjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v2beta1-mpijob + failurePolicy: Fail + name: mmpijob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v2beta1 + operations: + - CREATE + resources: + - mpijobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-ray-io-v1-raycluster + failurePolicy: Fail + name: mraycluster.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + resources: + - rayclusters + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-ray-io-v1-rayjob + failurePolicy: Fail + name: mrayjob.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + resources: + - rayjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-apps-v1-statefulset + failurePolicy: Fail + name: mstatefulset.kb.io + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - statefulsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kueue-x-k8s-io-v1beta1-clusterqueue + failurePolicy: Fail + name: mclusterqueue.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + resources: + - clusterqueues + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kueue-x-k8s-io-v1beta1-resourceflavor + failurePolicy: Fail + name: mresourceflavor.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + resources: + - resourceflavors + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kueue-x-k8s-io-v1beta1-workload + failurePolicy: Fail + name: mworkload.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + resources: + - workloads + sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate--v1-pod + failurePolicy: Fail + name: vpod.kb.io + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kueue-system + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - pods + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-apps-v1-deployment + failurePolicy: Fail + name: vdeployment.kb.io + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kueue-system + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - deployments + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-batch-v1-job + failurePolicy: Fail + name: vjob.kb.io + rules: + - apiGroups: + - batch + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - jobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-jobset-x-k8s-io-v1alpha2-jobset + failurePolicy: Fail + name: vjobset.kb.io + rules: + - apiGroups: + - jobset.x-k8s.io + apiVersions: + - v1alpha2 + operations: + - CREATE + - UPDATE + resources: + - jobsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v1-mxjob + failurePolicy: Fail + name: vmxjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - mxjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v1-paddlejob + failurePolicy: Fail + name: vpaddlejob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - paddlejobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v1-pytorchjob + failurePolicy: Fail + name: vpytorchjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - pytorchjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v1-tfjob + failurePolicy: Fail + name: vtfjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - tfjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v1-xgboostjob + failurePolicy: Fail + name: vxgboostjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - xgboostjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v2beta1-mpijob + failurePolicy: Fail + name: vmpijob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v2beta1 + operations: + - CREATE + - UPDATE + resources: + - mpijobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-ray-io-v1-raycluster + failurePolicy: Fail + name: vraycluster.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - rayclusters + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-ray-io-v1-rayjob + failurePolicy: Fail + name: vrayjob.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - rayjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-apps-v1-statefulset + failurePolicy: Fail + name: vstatefulset.kb.io + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - statefulsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kueue-x-k8s-io-v1beta1-clusterqueue + failurePolicy: Fail + name: vclusterqueue.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - clusterqueues + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kueue-x-k8s-io-v1alpha1-cohort + failurePolicy: Fail + name: vcohort.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - cohorts + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kueue-x-k8s-io-v1beta1-resourceflavor + failurePolicy: Fail + name: vresourceflavor.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - resourceflavors + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kueue-x-k8s-io-v1beta1-workload + failurePolicy: Fail + name: vworkload.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - workloads + - workloads/status + sideEffects: None diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.10.1.yaml b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.10.1.yaml new file mode 100644 index 0000000000..6cd24729b8 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.10.1.yaml @@ -0,0 +1,13175 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-system +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: admissionchecks.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: AdmissionCheck + listKind: AdmissionCheckList + plural: admissionchecks + singular: admissioncheck + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: AdmissionCheck is the Schema for the admissionchecks API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: AdmissionCheckSpec defines the desired state of AdmissionCheck + properties: + controllerName: + description: |- + controllerName identifies the controller that processes the AdmissionCheck, + not necessarily a Kubernetes Pod or Deployment name. Cannot be empty. + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + parameters: + description: |- + Parameters identifies a configuration with additional parameters for the + check. + properties: + apiGroup: + description: ApiGroup is the group for the resource being referenced. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is the type of the resource being referenced. + maxLength: 63 + pattern: ^(?i)[a-z]([-a-z0-9]*[a-z0-9])?$ + type: string + name: + description: Name is the name of the resource being referenced. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - apiGroup + - kind + - name + type: object + retryDelayMinutes: + default: 15 + description: |- + RetryDelayMinutes specifies how long to keep the workload suspended after + a failed check (after it transitioned to False). When the delay period has passed, the check + state goes to "Unknown". The default is 15 min. + Deprecated: retryDelayMinutes has already been deprecated since v0.8 and will be removed in v1beta2. + format: int64 + type: integer + required: + - controllerName + type: object + status: + description: AdmissionCheckStatus defines the observed state of AdmissionCheck + properties: + conditions: + description: |- + conditions hold the latest available observations of the AdmissionCheck + current state. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: clusterqueues.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: ClusterQueue + listKind: ClusterQueueList + plural: clusterqueues + shortNames: + - cq + singular: clusterqueue + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Cohort that this ClusterQueue belongs to + jsonPath: .spec.cohort + name: Cohort + type: string + - description: The queueing strategy used to prioritize workloads + jsonPath: .spec.queueingStrategy + name: Strategy + priority: 1 + type: string + - description: Number of pending workloads + jsonPath: .status.pendingWorkloads + name: Pending Workloads + type: integer + - description: Number of admitted workloads that haven't finished yet + jsonPath: .status.admittedWorkloads + name: Admitted Workloads + priority: 1 + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + description: ClusterQueue is the Schema for the clusterQueue API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ClusterQueueSpec defines the desired state of ClusterQueue + properties: + admissionChecks: + description: |- + admissionChecks lists the AdmissionChecks required by this ClusterQueue. + Cannot be used along with AdmissionCheckStrategy. + items: + type: string + type: array + admissionChecksStrategy: + description: |- + admissionCheckStrategy defines a list of strategies to determine which ResourceFlavors require AdmissionChecks. + This property cannot be used in conjunction with the 'admissionChecks' property. + properties: + admissionChecks: + description: admissionChecks is a list of strategies for AdmissionChecks + items: + description: AdmissionCheckStrategyRule defines rules for a + single AdmissionCheck + properties: + name: + description: name is an AdmissionCheck's name. + type: string + onFlavors: + description: |- + onFlavors is a list of ResourceFlavors' names that this AdmissionCheck should run for. + If empty, the AdmissionCheck will run for all workloads submitted to the ClusterQueue. + items: + description: ResourceFlavorReference is the name of the + ResourceFlavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + type: array + required: + - name + type: object + type: array + type: object + cohort: + description: |- + cohort that this ClusterQueue belongs to. CQs that belong to the + same cohort can borrow unused resources from each other. + + A CQ can be a member of a single borrowing cohort. A workload submitted + to a queue referencing this CQ can borrow quota from any CQ in the cohort. + Only quota for the [resource, flavor] pairs listed in the CQ can be + borrowed. + If empty, this ClusterQueue cannot borrow from any other ClusterQueue and + vice versa. + + A cohort is a name that links CQs together, but it doesn't reference any + object. + + Validation of a cohort name is equivalent to that of object names: + subdomain in DNS (RFC 1123). + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + fairSharing: + description: |- + fairSharing defines the properties of the ClusterQueue when participating in fair sharing. + The values are only relevant if fair sharing is enabled in the Kueue configuration. + properties: + weight: + anyOf: + - type: integer + - type: string + default: 1 + description: |- + weight gives a comparative advantage to this ClusterQueue when competing for unused + resources in the cohort against other ClusterQueues. + The share of a ClusterQueue is based on the dominant resource usage above nominal + quotas for each resource, divided by the weight. + Admission prioritizes scheduling workloads from ClusterQueues with the lowest share + and preempting workloads from the ClusterQueues with the highest share. + A zero weight implies infinite share value, meaning that this ClusterQueue will always + be at disadvantage against other ClusterQueues. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + flavorFungibility: + default: {} + description: |- + flavorFungibility defines whether a workload should try the next flavor + before borrowing or preempting in the flavor being evaluated. + properties: + whenCanBorrow: + default: Borrow + description: |- + whenCanBorrow determines whether a workload should try the next flavor + before borrowing in current flavor. The possible values are: + + - `Borrow` (default): allocate in current flavor if borrowing + is possible. + - `TryNextFlavor`: try next flavor even if the current + flavor has enough resources to borrow. + enum: + - Borrow + - TryNextFlavor + type: string + whenCanPreempt: + default: TryNextFlavor + description: |- + whenCanPreempt determines whether a workload should try the next flavor + before borrowing in current flavor. The possible values are: + + - `Preempt`: allocate in current flavor if it's possible to preempt some workloads. + - `TryNextFlavor` (default): try next flavor even if there are enough + candidates for preemption in the current flavor. + enum: + - Preempt + - TryNextFlavor + type: string + type: object + namespaceSelector: + description: |- + namespaceSelector defines which namespaces are allowed to submit workloads to + this clusterQueue. Beyond this basic support for policy, a policy agent like + Gatekeeper should be used to enforce more advanced policies. + Defaults to null which is a nothing selector (no namespaces eligible). + If set to an empty selector `{}`, then all namespaces are eligible. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + preemption: + default: {} + description: |- + preemption describes policies to preempt Workloads from this ClusterQueue + or the ClusterQueue's cohort. + + Preemption can happen in two scenarios: + + - When a Workload fits within the nominal quota of the ClusterQueue, but + the quota is currently borrowed by other ClusterQueues in the cohort. + Preempting Workloads in other ClusterQueues allows this ClusterQueue to + reclaim its nominal quota. + - When a Workload doesn't fit within the nominal quota of the ClusterQueue + and there are admitted Workloads in the ClusterQueue with lower priority. + + The preemption algorithm tries to find a minimal set of Workloads to + preempt to accomomdate the pending Workload, preempting Workloads with + lower priority first. + properties: + borrowWithinCohort: + default: {} + description: |- + borrowWithinCohort provides configuration to allow preemption within + cohort while borrowing. + properties: + maxPriorityThreshold: + description: |- + maxPriorityThreshold allows to restrict the set of workloads which + might be preempted by a borrowing workload, to only workloads with + priority less than or equal to the specified threshold priority. + When the threshold is not specified, then any workload satisfying the + policy can be preempted by the borrowing workload. + format: int32 + type: integer + policy: + default: Never + description: |- + policy determines the policy for preemption to reclaim quota within cohort while borrowing. + Possible values are: + - `Never` (default): do not allow for preemption, in other + ClusterQueues within the cohort, for a borrowing workload. + - `LowerPriority`: allow preemption, in other ClusterQueues + within the cohort, for a borrowing workload, but only if + the preempted workloads are of lower priority. + enum: + - Never + - LowerPriority + type: string + type: object + reclaimWithinCohort: + default: Never + description: |- + reclaimWithinCohort determines whether a pending Workload can preempt + Workloads from other ClusterQueues in the cohort that are using more than + their nominal quota. The possible values are: + + - `Never` (default): do not preempt Workloads in the cohort. + - `LowerPriority`: **Classic Preemption** if the pending Workload + fits within the nominal quota of its ClusterQueue, only preempt + Workloads in the cohort that have lower priority than the pending + Workload. **Fair Sharing** only preempt Workloads in the cohort that + have lower priority than the pending Workload and that satisfy the + fair sharing preemptionStategies. + - `Any`: **Classic Preemption** if the pending Workload fits within + the nominal quota of its ClusterQueue, preempt any Workload in the + cohort, irrespective of priority. **Fair Sharing** preempt Workloads + in the cohort that satisfy the fair sharing preemptionStrategies. + enum: + - Never + - LowerPriority + - Any + type: string + withinClusterQueue: + default: Never + description: |- + withinClusterQueue determines whether a pending Workload that doesn't fit + within the nominal quota for its ClusterQueue, can preempt active Workloads in + the ClusterQueue. The possible values are: + + - `Never` (default): do not preempt Workloads in the ClusterQueue. + - `LowerPriority`: only preempt Workloads in the ClusterQueue that have + lower priority than the pending Workload. + - `LowerOrNewerEqualPriority`: only preempt Workloads in the ClusterQueue that + either have a lower priority than the pending workload or equal priority + and are newer than the pending workload. + enum: + - Never + - LowerPriority + - LowerOrNewerEqualPriority + type: string + type: object + x-kubernetes-validations: + - message: reclaimWithinCohort=Never and borrowWithinCohort.Policy!=Never + rule: '!(self.reclaimWithinCohort == ''Never'' && has(self.borrowWithinCohort) + && self.borrowWithinCohort.policy != ''Never'')' + queueingStrategy: + default: BestEffortFIFO + description: |- + QueueingStrategy indicates the queueing strategy of the workloads + across the queues in this ClusterQueue. + Current Supported Strategies: + + - StrictFIFO: workloads are ordered strictly by creation time. + Older workloads that can't be admitted will block admitting newer + workloads even if they fit available quota. + - BestEffortFIFO: workloads are ordered by creation time, + however older workloads that can't be admitted will not block + admitting newer workloads that fit existing quota. + enum: + - StrictFIFO + - BestEffortFIFO + type: string + resourceGroups: + description: |- + resourceGroups describes groups of resources. + Each resource group defines the list of resources and a list of flavors + that provide quotas for these resources. + Each resource and each flavor can only form part of one resource group. + resourceGroups can be up to 16. + items: + properties: + coveredResources: + description: |- + coveredResources is the list of resources covered by the flavors in this + group. + Examples: cpu, memory, vendor.com/gpu. + The list cannot be empty and it can contain up to 16 resources. + items: + description: ResourceName is the name identifying various + resources in a ResourceList. + type: string + maxItems: 16 + minItems: 1 + type: array + flavors: + description: |- + flavors is the list of flavors that provide the resources of this group. + Typically, different flavors represent different hardware models + (e.g., gpu models, cpu architectures) or pricing models (on-demand vs spot + cpus). + Each flavor MUST list all the resources listed for this group in the same + order as the .resources field. + The list cannot be empty and it can contain up to 16 flavors. + items: + properties: + name: + description: |- + name of this flavor. The name should match the .metadata.name of a + ResourceFlavor. If a matching ResourceFlavor does not exist, the + ClusterQueue will have an Active condition set to False. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: |- + resources is the list of quotas for this flavor per resource. + There could be up to 16 resources. + items: + properties: + borrowingLimit: + anyOf: + - type: integer + - type: string + description: |- + borrowingLimit is the maximum amount of quota for the [flavor, resource] + combination that this ClusterQueue is allowed to borrow from the unused + quota of other ClusterQueues in the same cohort. + In total, at a given time, Workloads in a ClusterQueue can consume a + quantity of quota equal to nominalQuota+borrowingLimit, assuming the other + ClusterQueues in the cohort have enough unused quota. + If null, it means that there is no borrowing limit. + If not null, it must be non-negative. + borrowingLimit must be null if spec.cohort is empty. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + lendingLimit: + anyOf: + - type: integer + - type: string + description: |- + lendingLimit is the maximum amount of unused quota for the [flavor, resource] + combination that this ClusterQueue can lend to other ClusterQueues in the same cohort. + In total, at a given time, ClusterQueue reserves for its exclusive use + a quantity of quota equals to nominalQuota - lendingLimit. + If null, it means that there is no lending limit, meaning that + all the nominalQuota can be borrowed by other clusterQueues in the cohort. + If not null, it must be non-negative. + lendingLimit must be null if spec.cohort is empty. + This field is in beta stage and is enabled by default. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name of this resource. + type: string + nominalQuota: + anyOf: + - type: integer + - type: string + description: |- + nominalQuota is the quantity of this resource that is available for + Workloads admitted by this ClusterQueue at a point in time. + The nominalQuota must be non-negative. + nominalQuota should represent the resources in the cluster available for + running jobs (after discounting resources consumed by system components + and pods not managed by kueue). In an autoscaled cluster, nominalQuota + should account for resources that can be provided by a component such as + Kubernetes cluster-autoscaler. + + If the ClusterQueue belongs to a cohort, the sum of the quotas for each + (flavor, resource) combination defines the maximum quantity that can be + allocated by a ClusterQueue in the cohort. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + - nominalQuota + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - coveredResources + - flavors + type: object + x-kubernetes-validations: + - message: flavors must have the same number of resources as the + coveredResources + rule: self.flavors.all(x, size(x.resources) == size(self.coveredResources)) + maxItems: 16 + type: array + x-kubernetes-list-type: atomic + stopPolicy: + default: None + description: |- + stopPolicy - if set to a value different from None, the ClusterQueue is considered Inactive, no new reservation being + made. + + Depending on its value, its associated workloads will: + + - None - Workloads are admitted + - HoldAndDrain - Admitted workloads are evicted and Reserving workloads will cancel the reservation. + - Hold - Admitted workloads will run to completion and Reserving workloads will cancel the reservation. + enum: + - None + - Hold + - HoldAndDrain + type: string + type: object + x-kubernetes-validations: + - message: borrowingLimit must be nil when cohort is empty + rule: '!has(self.cohort) && has(self.resourceGroups) ? self.resourceGroups.all(rg, + rg.flavors.all(f, f.resources.all(r, !has(r.borrowingLimit)))) : true' + status: + description: ClusterQueueStatus defines the observed state of ClusterQueue + properties: + admittedWorkloads: + description: |- + admittedWorkloads is the number of workloads currently admitted to this + clusterQueue and haven't finished yet. + format: int32 + type: integer + conditions: + description: |- + conditions hold the latest available observations of the ClusterQueue + current state. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + fairSharing: + description: FairSharing contains the information about the current + status of fair sharing. + properties: + weightedShare: + description: |- + WeightedShare represent the maximum of the ratios of usage above nominal + quota to the lendable resources in the cohort, among all the resources + provided by the ClusterQueue, and divided by the weight. + If zero, it means that the usage of the ClusterQueue is below the nominal quota. + If the ClusterQueue has a weight of zero, this will return 9223372036854775807, + the maximum possible share value. + format: int64 + type: integer + required: + - weightedShare + type: object + flavorsReservation: + description: |- + flavorsReservation are the reserved quotas, by flavor, currently in use by the + workloads assigned to this ClusterQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: resources lists the quota usage for the resources + in this flavor. + items: + properties: + borrowed: + anyOf: + - type: integer + - type: string + description: |- + Borrowed is quantity of quota that is borrowed from the cohort. In other + words, it's the used quota that is over the nominalQuota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name of the resource + type: string + total: + anyOf: + - type: integer + - type: string + description: |- + total is the total quantity of used quota, including the amount borrowed + from the cohort. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + flavorsUsage: + description: |- + flavorsUsage are the used quotas, by flavor, currently in use by the + workloads admitted in this ClusterQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: resources lists the quota usage for the resources + in this flavor. + items: + properties: + borrowed: + anyOf: + - type: integer + - type: string + description: |- + Borrowed is quantity of quota that is borrowed from the cohort. In other + words, it's the used quota that is over the nominalQuota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name of the resource + type: string + total: + anyOf: + - type: integer + - type: string + description: |- + total is the total quantity of used quota, including the amount borrowed + from the cohort. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + pendingWorkloads: + description: |- + pendingWorkloads is the number of workloads currently waiting to be + admitted to this clusterQueue. + format: int32 + type: integer + pendingWorkloadsStatus: + description: |- + PendingWorkloadsStatus contains the information exposed about the current + status of the pending workloads in the cluster queue. + Deprecated: This field will be removed on v1beta2, use VisibilityOnDemand + (https://kueue.sigs.k8s.io/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand/) + instead. + properties: + clusterQueuePendingWorkload: + description: Head contains the list of top pending workloads. + items: + description: |- + ClusterQueuePendingWorkload contains the information identifying a pending workload + in the cluster queue. + properties: + name: + description: Name indicates the name of the pending workload. + type: string + namespace: + description: Namespace indicates the name of the pending + workload. + type: string + required: + - name + - namespace + type: object + type: array + x-kubernetes-list-type: atomic + lastChangeTime: + description: LastChangeTime indicates the time of the last change + of the structure. + format: date-time + type: string + required: + - lastChangeTime + type: object + reservingWorkloads: + description: |- + reservingWorkloads is the number of workloads currently reserving quota in this + clusterQueue. + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: cohorts.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: Cohort + listKind: CohortList + plural: cohorts + singular: cohort + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + Cohort is the Schema for the cohorts API. Using Hierarchical + Cohorts (any Cohort which has a parent) with Fair Sharing + results in undefined behavior in 0.9 + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CohortSpec defines the desired state of Cohort + properties: + parent: + description: |- + Parent references the name of the Cohort's parent, if + any. It satisfies one of three cases: + 1) Unset. This Cohort is the root of its Cohort tree. + 2) References a non-existent Cohort. We use default Cohort (no borrowing/lending limits). + 3) References an existent Cohort. + + If a cycle is created, we disable all members of the + Cohort, including ClusterQueues, until the cycle is + removed. We prevent further admission while the cycle + exists. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resourceGroups: + description: |- + ResourceGroups describes groupings of Resources and + Flavors. Each ResourceGroup defines a list of Resources + and a list of Flavors which provide quotas for these + Resources. Each Resource and each Flavor may only form part + of one ResourceGroup. There may be up to 16 ResourceGroups + within a Cohort. + + BorrowingLimit limits how much members of this Cohort + subtree can borrow from the parent subtree. + + LendingLimit limits how much members of this Cohort subtree + can lend to the parent subtree. + + Borrowing and Lending limits must only be set when the + Cohort has a parent. Otherwise, the Cohort create/update + will be rejected by the webhook. + items: + properties: + coveredResources: + description: |- + coveredResources is the list of resources covered by the flavors in this + group. + Examples: cpu, memory, vendor.com/gpu. + The list cannot be empty and it can contain up to 16 resources. + items: + description: ResourceName is the name identifying various + resources in a ResourceList. + type: string + maxItems: 16 + minItems: 1 + type: array + flavors: + description: |- + flavors is the list of flavors that provide the resources of this group. + Typically, different flavors represent different hardware models + (e.g., gpu models, cpu architectures) or pricing models (on-demand vs spot + cpus). + Each flavor MUST list all the resources listed for this group in the same + order as the .resources field. + The list cannot be empty and it can contain up to 16 flavors. + items: + properties: + name: + description: |- + name of this flavor. The name should match the .metadata.name of a + ResourceFlavor. If a matching ResourceFlavor does not exist, the + ClusterQueue will have an Active condition set to False. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: |- + resources is the list of quotas for this flavor per resource. + There could be up to 16 resources. + items: + properties: + borrowingLimit: + anyOf: + - type: integer + - type: string + description: |- + borrowingLimit is the maximum amount of quota for the [flavor, resource] + combination that this ClusterQueue is allowed to borrow from the unused + quota of other ClusterQueues in the same cohort. + In total, at a given time, Workloads in a ClusterQueue can consume a + quantity of quota equal to nominalQuota+borrowingLimit, assuming the other + ClusterQueues in the cohort have enough unused quota. + If null, it means that there is no borrowing limit. + If not null, it must be non-negative. + borrowingLimit must be null if spec.cohort is empty. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + lendingLimit: + anyOf: + - type: integer + - type: string + description: |- + lendingLimit is the maximum amount of unused quota for the [flavor, resource] + combination that this ClusterQueue can lend to other ClusterQueues in the same cohort. + In total, at a given time, ClusterQueue reserves for its exclusive use + a quantity of quota equals to nominalQuota - lendingLimit. + If null, it means that there is no lending limit, meaning that + all the nominalQuota can be borrowed by other clusterQueues in the cohort. + If not null, it must be non-negative. + lendingLimit must be null if spec.cohort is empty. + This field is in beta stage and is enabled by default. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name of this resource. + type: string + nominalQuota: + anyOf: + - type: integer + - type: string + description: |- + nominalQuota is the quantity of this resource that is available for + Workloads admitted by this ClusterQueue at a point in time. + The nominalQuota must be non-negative. + nominalQuota should represent the resources in the cluster available for + running jobs (after discounting resources consumed by system components + and pods not managed by kueue). In an autoscaled cluster, nominalQuota + should account for resources that can be provided by a component such as + Kubernetes cluster-autoscaler. + + If the ClusterQueue belongs to a cohort, the sum of the quotas for each + (flavor, resource) combination defines the maximum quantity that can be + allocated by a ClusterQueue in the cohort. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + - nominalQuota + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - coveredResources + - flavors + type: object + x-kubernetes-validations: + - message: flavors must have the same number of resources as the + coveredResources + rule: self.flavors.all(x, size(x.resources) == size(self.coveredResources)) + maxItems: 16 + type: array + x-kubernetes-list-type: atomic + type: object + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: localqueues.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: LocalQueue + listKind: LocalQueueList + plural: localqueues + shortNames: + - queue + - queues + - lq + singular: localqueue + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Backing ClusterQueue + jsonPath: .spec.clusterQueue + name: ClusterQueue + type: string + - description: Number of pending workloads + jsonPath: .status.pendingWorkloads + name: Pending Workloads + type: integer + - description: Number of admitted workloads that haven't finished yet. + jsonPath: .status.admittedWorkloads + name: Admitted Workloads + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + description: LocalQueue is the Schema for the localQueues API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: LocalQueueSpec defines the desired state of LocalQueue + properties: + clusterQueue: + description: clusterQueue is a reference to a clusterQueue that backs + this localQueue. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + stopPolicy: + default: None + description: |- + stopPolicy - if set to a value different from None, the LocalQueue is considered Inactive, + no new reservation being made. + + Depending on its value, its associated workloads will: + + - None - Workloads are admitted + - HoldAndDrain - Admitted workloads are evicted and Reserving workloads will cancel the reservation. + - Hold - Admitted workloads will run to completion and Reserving workloads will cancel the reservation. + enum: + - None + - Hold + - HoldAndDrain + type: string + type: object + status: + description: LocalQueueStatus defines the observed state of LocalQueue + properties: + admittedWorkloads: + description: |- + admittedWorkloads is the number of workloads in this LocalQueue + admitted to a ClusterQueue and that haven't finished yet. + format: int32 + type: integer + conditions: + description: |- + Conditions hold the latest available observations of the LocalQueue + current state. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + flavorUsage: + description: |- + flavorsUsage are the used quotas, by flavor currently in use by the + workloads assigned to this LocalQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: resources lists the quota usage for the resources + in this flavor. + items: + properties: + name: + description: name of the resource. + type: string + total: + anyOf: + - type: integer + - type: string + description: total is the total quantity of used quota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + flavors: + description: flavors lists all currently available ResourceFlavors + in specified ClusterQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + nodeLabels: + additionalProperties: + type: string + description: |- + nodeLabels are labels that associate the ResourceFlavor with Nodes that + have the same labels. + maxProperties: 8 + type: object + x-kubernetes-map-type: atomic + nodeTaints: + description: |- + nodeTaints are taints that the nodes associated with this ResourceFlavor + have. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to + a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint + key. + type: string + required: + - effect + - key + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + resources: + description: resources used in the flavor. + items: + description: ResourceName is the name identifying various + resources in a ResourceList. + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + flavorsReservation: + description: |- + flavorsReservation are the reserved quotas, by flavor currently in use by the + workloads assigned to this LocalQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: resources lists the quota usage for the resources + in this flavor. + items: + properties: + name: + description: name of the resource. + type: string + total: + anyOf: + - type: integer + - type: string + description: total is the total quantity of used quota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + pendingWorkloads: + description: PendingWorkloads is the number of Workloads in the LocalQueue + not yet admitted to a ClusterQueue + format: int32 + type: integer + reservingWorkloads: + description: |- + reservingWorkloads is the number of workloads in this LocalQueue + reserving quota in a ClusterQueue and that haven't finished yet. + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: multikueueclusters.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: MultiKueueCluster + listKind: MultiKueueClusterList + plural: multikueueclusters + singular: multikueuecluster + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: MultiKueueCluster is the Schema for the multikueue API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + kubeConfig: + description: Information how to connect to the cluster. + properties: + location: + description: |- + Location of the KubeConfig. + + If LocationType is Secret then Location is the name of the secret inside the namespace in + which the kueue controller manager is running. The config should be stored in the "kubeconfig" key. + type: string + locationType: + default: Secret + description: Type of the KubeConfig location. + enum: + - Secret + - Path + type: string + required: + - location + - locationType + type: object + required: + - kubeConfig + type: object + status: + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: multikueueconfigs.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: MultiKueueConfig + listKind: MultiKueueConfigList + plural: multikueueconfigs + singular: multikueueconfig + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: MultiKueueConfig is the Schema for the multikueue API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: MultiKueueConfigSpec defines the desired state of MultiKueueConfig + properties: + clusters: + description: List of MultiKueueClusters names where the workloads + from the ClusterQueue should be distributed. + items: + type: string + maxItems: 10 + minItems: 1 + type: array + x-kubernetes-list-type: set + required: + - clusters + type: object + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: provisioningrequestconfigs.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: ProvisioningRequestConfig + listKind: ProvisioningRequestConfigList + plural: provisioningrequestconfigs + singular: provisioningrequestconfig + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: ProvisioningRequestConfig is the Schema for the provisioningrequestconfig + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ProvisioningRequestConfigSpec defines the desired state of + ProvisioningRequestConfig + properties: + managedResources: + description: |- + managedResources contains the list of resources managed by the autoscaling. + + If empty, all resources are considered managed. + + If not empty, the ProvisioningRequest will contain only the podsets that are + requesting at least one of them. + + If none of the workloads podsets is requesting at least a managed resource, + the workload is considered ready. + items: + description: ResourceName is the name identifying various resources + in a ResourceList. + type: string + maxItems: 100 + type: array + x-kubernetes-list-type: set + parameters: + additionalProperties: + description: Parameter is limited to 255 characters. + maxLength: 255 + type: string + description: Parameters contains all other parameters classes may + require. + maxProperties: 100 + type: object + provisioningClassName: + description: |- + ProvisioningClassName describes the different modes of provisioning the resources. + Check autoscaling.x-k8s.io ProvisioningRequestSpec.ProvisioningClassName for details. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + retryStrategy: + default: + backoffBaseSeconds: 60 + backoffLimitCount: 3 + backoffMaxSeconds: 1800 + description: |- + retryStrategy defines strategy for retrying ProvisioningRequest. + If null, then the default configuration is applied with the following parameter values: + backoffLimitCount: 3 + backoffBaseSeconds: 60 - 1 min + backoffMaxSeconds: 1800 - 30 mins + + To switch off retry mechanism + set retryStrategy.backoffLimitCount to 0. + properties: + backoffBaseSeconds: + default: 60 + description: |- + BackoffBaseSeconds defines the base for the exponential backoff for + re-queuing an evicted workload. + + Defaults to 60. + format: int32 + type: integer + backoffLimitCount: + default: 3 + description: |- + BackoffLimitCount defines the maximum number of re-queuing retries. + Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). + + Every backoff duration is about "b*2^(n-1)+Rand" where: + - "b" represents the base set by "BackoffBaseSeconds" parameter, + - "n" represents the "workloadStatus.requeueState.count", + - "Rand" represents the random jitter. + During this time, the workload is taken as an inadmissible and + other workloads will have a chance to be admitted. + By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...). + + Defaults to 3. + format: int32 + type: integer + backoffMaxSeconds: + default: 1800 + description: |- + BackoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload. + + Defaults to 1800. + format: int32 + type: integer + type: object + required: + - provisioningClassName + type: object + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: resourceflavors.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: ResourceFlavor + listKind: ResourceFlavorList + plural: resourceflavors + shortNames: + - flavor + - flavors + - rf + singular: resourceflavor + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: ResourceFlavor is the Schema for the resourceflavors API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ResourceFlavorSpec defines the desired state of the ResourceFlavor + properties: + nodeLabels: + additionalProperties: + type: string + description: |- + nodeLabels are labels that associate the ResourceFlavor with Nodes that + have the same labels. + When a Workload is admitted, its podsets can only get assigned + ResourceFlavors whose nodeLabels match the nodeSelector and nodeAffinity + fields. + Once a ResourceFlavor is assigned to a podSet, the ResourceFlavor's + nodeLabels should be injected into the pods of the Workload by the + controller that integrates with the Workload object. + + nodeLabels can be up to 8 elements. + maxProperties: 8 + type: object + x-kubernetes-map-type: atomic + nodeTaints: + description: |- + nodeTaints are taints that the nodes associated with this ResourceFlavor + have. + Workloads' podsets must have tolerations for these nodeTaints in order to + get assigned this ResourceFlavor during admission. + + An example of a nodeTaint is + cloud.provider.com/preemptible="true":NoSchedule + + nodeTaints can be up to 8 elements. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: 'supported taint effect values: ''NoSchedule'', ''PreferNoSchedule'', + ''NoExecute''' + rule: self.all(x, x.effect in ['NoSchedule', 'PreferNoSchedule', + 'NoExecute']) + tolerations: + description: |- + tolerations are extra tolerations that will be added to the pods admitted in + the quota associated with this resource flavor. + + An example of a toleration is + cloud.provider.com/preemptible="true":NoSchedule + + tolerations can be up to 8 elements. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: operator must be Exists when 'key' is empty, which means + 'match all values and all keys' + rule: 'self.all(x, !has(x.key) ? x.operator == ''Exists'' : true)' + - message: effect must be 'NoExecute' when 'tolerationSeconds' is + set + rule: 'self.all(x, has(x.tolerationSeconds) ? x.effect == ''NoExecute'' + : true)' + - message: 'supported toleration values: ''Equal''(default), ''Exists''' + rule: self.all(x, !has(x.operator) || x.operator in ['Equal', 'Exists']) + - message: a value must be empty when 'operator' is 'Exists' + rule: 'self.all(x, has(x.operator) && x.operator == ''Exists'' ? + !has(x.value) : true)' + - message: 'supported taint effect values: ''NoSchedule'', ''PreferNoSchedule'', + ''NoExecute''' + rule: self.all(x, !has(x.effect) || x.effect in ['NoSchedule', 'PreferNoSchedule', + 'NoExecute']) + topologyName: + description: |- + topologyName indicates topology for the TAS ResourceFlavor. + When specified, it enables scraping of the topology information from the + nodes matching to the Resource Flavor node labels. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + type: object + x-kubernetes-validations: + - message: at least one nodeLabel is required when topology is set + rule: '!has(self.topologyName) || self.nodeLabels.size() >= 1' + - message: resourceFlavorSpec are immutable when topologyName is set + rule: '!has(oldSelf.topologyName) || self == oldSelf' + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: topologies.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: Topology + listKind: TopologyList + plural: topologies + singular: topology + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Topology is the Schema for the topology API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: TopologySpec defines the desired state of Topology + properties: + levels: + description: levels define the levels of topology. + items: + description: TopologyLevel defines the desired state of TopologyLevel + properties: + nodeLabel: + description: |- + nodeLabel indicates the name of the node label for a specific topology + level. + + Examples: + - cloud.provider.com/topology-block + - cloud.provider.com/topology-rack + maxLength: 316 + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - nodeLabel + type: object + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + - message: must be unique + rule: size(self.filter(i, size(self.filter(j, j == i)) > 1)) == + 0 + - message: the kubernetes.io/hostname label can only be used at the + lowest level of topology + rule: size(self.filter(i, i.nodeLabel == 'kubernetes.io/hostname')) + == 0 || self[size(self) - 1].nodeLabel == 'kubernetes.io/hostname' + required: + - levels + type: object + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: workloadpriorityclasses.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: WorkloadPriorityClass + listKind: WorkloadPriorityClassList + plural: workloadpriorityclasses + singular: workloadpriorityclass + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Value of workloadPriorityClass's Priority + jsonPath: .value + name: Value + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + description: WorkloadPriorityClass is the Schema for the workloadPriorityClass + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + description: + description: |- + description is an arbitrary string that usually provides guidelines on + when this workloadPriorityClass should be used. + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + value: + description: |- + value represents the integer value of this workloadPriorityClass. This is the actual priority that workloads + receive when jobs have the name of this class in their workloadPriorityClass label. + Changing the value of workloadPriorityClass doesn't affect the priority of workloads that were already created. + format: int32 + type: integer + required: + - value + type: object + served: true + storage: true + subresources: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + controller-gen.kubebuilder.io/version: v0.16.5 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: workloads.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: Workload + listKind: WorkloadList + plural: workloads + shortNames: + - wl + singular: workload + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Name of the queue this workload was submitted to + jsonPath: .spec.queueName + name: Queue + type: string + - description: Name of the ClusterQueue where the workload is reserving quota + jsonPath: .status.admission.clusterQueue + name: Reserved in + type: string + - description: Admission status + jsonPath: .status.conditions[?(@.type=='Admitted')].status + name: Admitted + type: string + - description: Workload finished + jsonPath: .status.conditions[?(@.type=='Finished')].status + name: Finished + type: string + - description: Time this workload was created + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Workload is the Schema for the workloads API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: WorkloadSpec defines the desired state of Workload + properties: + active: + default: true + description: |- + Active determines if a workload can be admitted into a queue. + Changing active from true to false will evict any running workloads. + Possible values are: + + - false: indicates that a workload should never be admitted and evicts running workloads + - true: indicates that a workload can be evaluated for admission into it's respective queue. + + Defaults to true + type: boolean + maximumExecutionTimeSeconds: + description: |- + maximumExecutionTimeSeconds if provided, determines the maximum time, in seconds, + the workload can be admitted before it's automatically deactivated. + + If unspecified, no execution time limit is enforced on the Workload. + format: int32 + minimum: 1 + type: integer + podSets: + description: |- + podSets is a list of sets of homogeneous pods, each described by a Pod spec + and a count. + There must be at least one element and at most 8. + podSets cannot be changed. + items: + properties: + count: + default: 1 + description: count is the number of pods for the spec. + format: int32 + minimum: 0 + type: integer + minCount: + description: |- + minCount is the minimum number of pods for the spec acceptable + if the workload supports partial admission. + + If not provided, partial admission for the current PodSet is not + enabled. + + Only one podSet within the workload can use this. + + This is an alpha field and requires enabling PartialAdmission feature gate. + format: int32 + minimum: 1 + type: integer + name: + default: main + description: name is the PodSet name. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + template: + description: |- + template is the Pod template. + + The only allowed fields in template.metadata are labels and annotations. + + If requests are omitted for a container or initContainer, + they default to the limits if they are explicitly specified for the + container or initContainer. + + During admission, the rules in nodeSelector and + nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution that match + the keys in the nodeLabels from the ResourceFlavors considered for this + Workload are used to filter the ResourceFlavors that can be assigned to + this podSet. + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling + rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, + in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules + (e.g. co-locate this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the same + node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates + whether a service account token should be automatically + mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container that you + want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if value + is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name of + a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of + a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS resolver + options of a pod. + properties: + name: + description: Required. + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if value + is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed for ephemeral + containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral + containers. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + This may only be set for init containers. You cannot set this field on + ephemeral containers. + type: string + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name of + a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of + a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + If this option is set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + of that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container that you + want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if value + is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name of + a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of + a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the reference + to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition + in the pod's condition list with matching type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated to a + Pod to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that + applies to the container. + type: string + role: + description: Role is a SELinux role label that + applies to the container. + type: string + type: + description: Type is a SELinux type label that + applies to the container. + type: string + user: + description: User is a SELinux user label that + applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter + to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies how + to spread matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume in a + pod that may be accessed by any container in the + pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data + Disk mount on the host and bind mount to the + pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching + mode: None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data + disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk + in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: + multiple blob disks per storage account Dedicated: + single blob disk per storage account Managed: + azure managed data disk (only in managed + availability set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File + Service mount on the host and bind mount to + the pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret + that contains Azure Storage Account Name + and Key + type: string + shareName: + description: shareName is the azure share + Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount + on the host that shares a pod's lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as the + mounted root, rather than the full Ceph + tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap + that should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the + ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) + represents ephemeral storage that is handled + by certain external CSI drivers (Beta feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API + about the pod that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API + volume file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file to + be created. Must not be absolute or + contain the ''..'' path. Must be utf-8 + encoded. The first item of the relative + path must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query + over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding + reference to the PersistentVolume + backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource + that is attached to a kubelet's host machine + and then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target lun + number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target + worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver + to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field + holds extra command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume + attached to a kubelet's host machine. This depends + on the Flocker control service being running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the + dataset. This is unique identifier of a + Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for + the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether + support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether + support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified + Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun + number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret + for iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a + PhotonController persistent disk attached and + mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx + volume attached and mounted on kubelets host + machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies + a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources + secrets, configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from + the volume root to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about + the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key + to a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether + the ConfigMap or its keys must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information + about the downwardAPI data to project + properties: + items: + description: Items is a list of + DownwardAPIVolume file + items: + description: DownwardAPIVolumeFile + represents information to create + the file containing the pod + field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only + annotations, labels, name, + namespace and uid are supported.' + properties: + apiVersion: + description: Version of + the schema the FieldPath + is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the + field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path + is the relative path name + of the file to be created. + Must not be absolute or + contain the ''..'' path. + Must be utf-8 encoded. The + first item of the relative + path must not start with + ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies + the output format of + the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information about + the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key + to a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify + whether the Secret or its key + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is + information about the serviceAccountToken + data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: quobyte represents a Quobyte mount + on the host that shares a pod's lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references + an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent + volume attached and mounted on Kubernetes nodes. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of + the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name + of the ScaleIO Protection Domain for the + configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default + false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage + Pool associated with the protection domain. + type: string + system: + description: system is the name of the storage + system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether + the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS + volume attached and mounted on Kubernetes nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere + volume attached and mounted on kubelets host + machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage + Policy Based Management (SPBM) profile ID + associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage + Policy Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies + vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object + topologyRequest: + description: topologyRequest defines the topology request for + the PodSet. + properties: + podIndexLabel: + description: |- + PodIndexLabel indicates the name of the label indexing the pods. + For example, in the context of + - kubernetes job this is: kubernetes.io/job-completion-index + - JobSet: kubernetes.io/job-completion-index (inherited from Job) + - Kubeflow: training.kubeflow.org/replica-index + type: string + preferred: + description: |- + preferred indicates the topology level preferred by the PodSet, as + indicated by the `kueue.x-k8s.io/podset-preferred-topology` PodSet + annotation. + type: string + required: + description: |- + required indicates the topology level required by the PodSet, as + indicated by the `kueue.x-k8s.io/podset-required-topology` PodSet + annotation. + type: string + subGroupCount: + description: |- + SubGroupIndexLabel indicates the count of replicated Jobs (groups) within a PodSet. + For example, in the context of JobSet this value is read from jobset.sigs.k8s.io/replicatedjob-replicas. + format: int32 + type: integer + subGroupIndexLabel: + description: |- + SubGroupIndexLabel indicates the name of the label indexing the instances of replicated Jobs (groups) + within a PodSet. For example, in the context of JobSet this is jobset.sigs.k8s.io/job-index. + type: string + type: object + required: + - count + - template + type: object + x-kubernetes-validations: + - message: minCount should be positive and less or equal to count + rule: 'has(self.minCount) ? self.minCount <= self.count : true' + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + priority: + description: |- + Priority determines the order of access to the resources managed by the + ClusterQueue where the workload is queued. + The priority value is populated from PriorityClassName. + The higher the value, the higher the priority. + If priorityClassName is specified, priority must not be null. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the workload's priority. + "system-node-critical" and "system-cluster-critical" are two special + keywords which indicate the highest priorities with the former being + the highest priority. Any other name must be defined by creating a + PriorityClass object with that name. If not specified, the workload + priority will be default or zero if there is no default. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + priorityClassSource: + default: "" + description: |- + priorityClassSource determines whether the priorityClass field refers to a pod PriorityClass or kueue.x-k8s.io/workloadpriorityclass. + Workload's PriorityClass can accept the name of a pod priorityClass or a workloadPriorityClass. + When using pod PriorityClass, a priorityClassSource field has the scheduling.k8s.io/priorityclass value. + enum: + - kueue.x-k8s.io/workloadpriorityclass + - scheduling.k8s.io/priorityclass + - "" + type: string + queueName: + description: |- + queueName is the name of the LocalQueue the Workload is associated with. + queueName cannot be changed while .status.admission is not null. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - podSets + type: object + x-kubernetes-validations: + - message: priority should not be nil when priorityClassName is set + rule: 'has(self.priorityClassName) ? has(self.priority) : true' + status: + description: WorkloadStatus defines the observed state of Workload + properties: + accumulatedPastExexcutionTimeSeconds: + description: |- + accumulatedPastExexcutionTimeSeconds holds the total time, in seconds, the workload spent + in Admitted state, in the previous `Admit` - `Evict` cycles. + format: int32 + type: integer + admission: + description: |- + admission holds the parameters of the admission of the workload by a + ClusterQueue. admission can be set back to null, but its fields cannot be + changed once set. + properties: + clusterQueue: + description: clusterQueue is the name of the ClusterQueue that + admitted this workload. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + podSetAssignments: + description: PodSetAssignments hold the admission results for + each of the .spec.podSets entries. + items: + properties: + count: + description: |- + count is the number of pods taken into account at admission time. + This field will not change in case of quota reclaim. + Value could be missing for Workloads created before this field was added, + in that case spec.podSets[*].count value will be used. + format: int32 + minimum: 0 + type: integer + flavors: + additionalProperties: + description: ResourceFlavorReference is the name of the + ResourceFlavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + description: Flavors are the flavors assigned to the workload + for each resource. + type: object + name: + default: main + description: Name is the name of the podSet. It should match + one of the names in .spec.podSets. + maxLength: 63 + pattern: ^(?i)[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + resourceUsage: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + resourceUsage keeps track of the total resources all the pods in the podset need to run. + + Beside what is provided in podSet's specs, this calculation takes into account + the LimitRange defaults and RuntimeClass overheads at the moment of admission. + This field will not change in case of quota reclaim. + type: object + topologyAssignment: + description: |- + topologyAssignment indicates the topology assignment divided into + topology domains corresponding to the lowest level of the topology. + The assignment specifies the number of Pods to be scheduled per topology + domain and specifies the node selectors for each topology domain, in the + following way: the node selector keys are specified by the levels field + (same for all domains), and the corresponding node selector value is + specified by the domains.values subfield. If the TopologySpec.Levels field contains + "kubernetes.io/hostname" label, topologyAssignment will contain data only for + this label, and omit higher levels in the topology + + Example: + + topologyAssignment: + levels: + - cloud.provider.com/topology-block + - cloud.provider.com/topology-rack + domains: + - values: [block-1, rack-1] + count: 4 + - values: [block-1, rack-2] + count: 2 + + Here: + - 4 Pods are to be scheduled on nodes matching the node selector: + cloud.provider.com/topology-block: block-1 + cloud.provider.com/topology-rack: rack-1 + - 2 Pods are to be scheduled on nodes matching the node selector: + cloud.provider.com/topology-block: block-1 + cloud.provider.com/topology-rack: rack-2 + + Example: + Below there is an equivalent of the above example assuming, Topology + object defines kubernetes.io/hostname as the lowest level in topology. + Hence we omit higher level of topologies, since the hostname label + is sufficient to explicitly identify a proper node. + + topologyAssignment: + levels: + - kubernetes.io/hostname + domains: + - values: [hostname-1] + count: 4 + - values: [hostname-2] + count: 2 + properties: + domains: + description: |- + domains is a list of topology assignments split by topology domains at + the lowest level of the topology. + items: + properties: + count: + description: |- + count indicates the number of Pods to be scheduled in the topology + domain indicated by the values field. + format: int32 + minimum: 1 + type: integer + values: + description: |- + values is an ordered list of node selector values describing a topology + domain. The values correspond to the consecutive topology levels, from + the highest to the lowest. + items: + type: string + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + required: + - count + - values + type: object + type: array + levels: + description: |- + levels is an ordered list of keys denoting the levels of the assigned + topology (i.e. node label keys), from the highest to the lowest level of + the topology. + items: + type: string + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + required: + - domains + - levels + type: object + required: + - name + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - clusterQueue + - podSetAssignments + type: object + admissionChecks: + description: admissionChecks list all the admission checks required + by the workload and the current status + items: + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + name: + description: name identifies the admission check. + maxLength: 316 + type: string + podSetUpdates: + items: + description: |- + PodSetUpdate contains a list of pod set modifications suggested by AdmissionChecks. + The modifications should be additive only - modifications of already existing keys + or having the same key provided by multiple AdmissionChecks is not allowed and will + result in failure during workload admission. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + name: + description: Name of the PodSet to modify. Should match + to one of the Workload's PodSets. + type: string + nodeSelector: + additionalProperties: + type: string + type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + maxItems: 8 + type: array + x-kubernetes-validations: + - message: operator must be Exists when 'key' is empty, + which means 'match all values and all keys' + rule: 'self.all(x, !has(x.key) ? x.operator == ''Exists'' + : true)' + - message: effect must be 'NoExecute' when 'tolerationSeconds' + is set + rule: 'self.all(x, has(x.tolerationSeconds) ? x.effect + == ''NoExecute'' : true)' + - message: 'supported toleration values: ''Equal''(default), + ''Exists''' + rule: self.all(x, !has(x.operator) || x.operator in + ['Equal', 'Exists']) + - message: a value must be empty when 'operator' is 'Exists' + rule: 'self.all(x, has(x.operator) && x.operator == + ''Exists'' ? !has(x.value) : true)' + - message: 'supported taint effect values: ''NoSchedule'', + ''PreferNoSchedule'', ''NoExecute''' + rule: self.all(x, !has(x.effect) || x.effect in ['NoSchedule', + 'PreferNoSchedule', 'NoExecute']) + required: + - name + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + state: + description: state of the admissionCheck, one of Pending, Ready, + Retry, Rejected + enum: + - Pending + - Ready + - Retry + - Rejected + type: string + required: + - lastTransitionTime + - message + - name + - state + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + conditions: + description: |- + conditions hold the latest available observations of the Workload + current state. + + The type of the condition could be: + + - Admitted: the Workload was admitted through a ClusterQueue. + - Finished: the associated workload finished running (failed or succeeded). + - PodsReady: at least `.spec.podSets[*].count` Pods are ready or have + succeeded. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + reclaimablePods: + description: |- + reclaimablePods keeps track of the number pods within a podset for which + the resource reservation is no longer needed. + items: + properties: + count: + description: count is the number of pods for which the requested + resources are no longer needed. + format: int32 + minimum: 0 + type: integer + name: + description: name is the PodSet name. + type: string + required: + - count + - name + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + requeueState: + description: |- + requeueState holds the re-queue state + when a workload meets Eviction with PodsReadyTimeout reason. + properties: + count: + description: |- + count records the number of times a workload has been re-queued + When a deactivated (`.spec.activate`=`false`) workload is reactivated (`.spec.activate`=`true`), + this count would be reset to null. + format: int32 + minimum: 0 + type: integer + requeueAt: + description: |- + requeueAt records the time when a workload will be re-queued. + When a deactivated (`.spec.activate`=`false`) workload is reactivated (`.spec.activate`=`true`), + this time would be reset to null. + format: date-time + type: string + type: object + resourceRequests: + description: |- + resourceRequests provides a detailed view of the resources that were + requested by a non-admitted workload when it was considered for admission. + If admission is non-null, resourceRequests will be empty because + admission.resourceUsage contains the detailed information. + items: + properties: + name: + default: main + description: name is the name of the podSet. It should match + one of the names in .spec.podSets. + maxLength: 63 + pattern: ^(?i)[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + resources is the total resources all the pods in the podset need to run. + + Beside what is provided in podSet's specs, this value also takes into account + the LimitRange defaults and RuntimeClass overheads at the moment of consideration + and the application of resource.excludeResourcePrefixes and resource.transformations. + type: object + required: + - name + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + type: object + x-kubernetes-validations: + - message: podSetAssignments must have the same number of podSets as the spec + rule: 'has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, + c.type == ''QuotaReserved'' && c.status == ''True'') && has(self.status.admission) + ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) + : true' + - message: field is immutable + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, + c.type == ''QuotaReserved'' && c.status == ''True'')) ? (oldSelf.spec.priorityClassSource + == self.spec.priorityClassSource) : true' + - message: field is immutable + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, + c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassName) + && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName + == self.spec.priorityClassName) : true' + - message: field is immutable + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, + c.type == ''QuotaReserved'' && c.status == ''True'')) && (has(self.status) + && has(self.status.conditions) && self.status.conditions.exists(c, c.type + == ''QuotaReserved'' && c.status == ''True'')) && has(oldSelf.spec.queueName) + && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName + : true' + - message: maximumExecutionTimeSeconds is immutable while admitted + rule: ((has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, + c.type == 'Admitted' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) + && self.status.conditions.exists(c, c.type == 'Admitted' && c.status == + 'True')))?((has(oldSelf.spec.maximumExecutionTimeSeconds)?oldSelf.spec.maximumExecutionTimeSeconds:0) + == (has(self.spec.maximumExecutionTimeSeconds)?self.spec.maximumExecutionTimeSeconds:0)):true + served: true + storage: true + subresources: + status: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-controller-manager + namespace: kueue-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-leader-election-role + namespace: kueue-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.kueue.x-k8s.io/batch-admin: "true" +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-batch-admin-role +--- +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.kueue.x-k8s.io/batch-user: "true" +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-batch-user-role +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-clusterqueue-editor-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-clusterqueue-viewer-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues + verbs: + - get + - list + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-job-editor-role +rules: +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch + resources: + - jobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-job-viewer-role +rules: +- apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch +- apiGroups: + - batch + resources: + - jobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-jobset-editor-role +rules: +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-jobset-viewer-role +rules: +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets + verbs: + - get + - list + - watch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-localqueue-editor-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - localqueues + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - localqueues/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-localqueue-viewer-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - localqueues + verbs: + - get + - list + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - localqueues/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-manager-role +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - watch +- apiGroups: + - "" + resources: + - limitranges + - namespaces + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods/finalizers + verbs: + - get + - update +- apiGroups: + - "" + resources: + - pods/status + verbs: + - get + - patch +- apiGroups: + - "" + resources: + - podtemplates + verbs: + - create + - delete + - get + - list + - update + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - get + - list + - update + - watch +- apiGroups: + - apps + resources: + - replicasets + - statefulsets + verbs: + - get + - list + - watch +- apiGroups: + - autoscaling.x-k8s.io + resources: + - provisioningrequests + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - autoscaling.x-k8s.io + resources: + - provisioningrequests/status + verbs: + - get +- apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - batch + resources: + - jobs/finalizers + - jobs/status + verbs: + - get + - patch + - update +- apiGroups: + - flowcontrol.apiserver.k8s.io + resources: + - flowschemas + - prioritylevelconfigurations + verbs: + - list + - watch +- apiGroups: + - flowcontrol.apiserver.k8s.io + resources: + - flowschemas/status + verbs: + - patch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/finalizers + verbs: + - get + - update +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/status + verbs: + - get + - patch + - update +- apiGroups: + - kubeflow.org + resources: + - mpijobs + - mxjobs + - paddlejobs + - pytorchjobs + - tfjobs + - xgboostjobs + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - mpijobs/finalizers + - mxjobs/finalizers + - mxjobs/status + - paddlejobs/finalizers + - pytorchjobs/finalizers + - tfjobs/finalizers + - xgboostjobs/finalizers + verbs: + - get + - update +- apiGroups: + - kubeflow.org + resources: + - mpijobs/status + - paddlejobs/status + - pytorchjobs/status + - tfjobs/status + - xgboostjobs/status + verbs: + - get + - patch + - update +- apiGroups: + - kueue.x-k8s.io + resources: + - admissionchecks + - clusterqueues + - cohorts + - localqueues + - workloads + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - admissionchecks/finalizers + - clusterqueues/finalizers + - localqueues/finalizers + - resourceflavors/finalizers + - topologies/finalizers + - workloads/finalizers + verbs: + - update +- apiGroups: + - kueue.x-k8s.io + resources: + - admissionchecks/status + - clusterqueues/status + - localqueues/status + - multikueueclusters/status + - workloads/status + verbs: + - get + - patch + - update +- apiGroups: + - kueue.x-k8s.io + resources: + - multikueueclusters + - multikueueconfigs + - provisioningrequestconfigs + - workloadpriorityclasses + verbs: + - get + - list + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - resourceflavors + verbs: + - delete + - get + - list + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - topologies + verbs: + - get + - list + - update + - watch +- apiGroups: + - node.k8s.io + resources: + - runtimeclasses + verbs: + - get + - list + - watch +- apiGroups: + - ray.io + resources: + - rayclusters + - rayjobs + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - ray.io + resources: + - rayclusters/finalizers + - rayclusters/status + - rayjobs/finalizers + - rayjobs/status + verbs: + - get + - update +- apiGroups: + - scheduling.k8s.io + resources: + - priorityclasses + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-mpijob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - mpijobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - mpijobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-mpijob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - mpijobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - mpijobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-mxjob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - mxjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - mxjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-mxjob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - mxjobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - mxjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-paddlejob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - paddlejobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - paddlejobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-paddlejob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - paddlejobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - paddlejobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-pending-workloads-cq-viewer-role +rules: +- apiGroups: + - visibility.kueue.x-k8s.io + resources: + - clusterqueues/pendingworkloads + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-pending-workloads-lq-viewer-role +rules: +- apiGroups: + - visibility.kueue.x-k8s.io + resources: + - localqueues/pendingworkloads + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-pytorchjob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-pytorchjob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-raycluster-editor-role +rules: +- apiGroups: + - ray.io + resources: + - rayclusters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ray.io + resources: + - rayclusters/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-raycluster-viewer-role +rules: +- apiGroups: + - ray.io + resources: + - rayclusters + verbs: + - get + - list + - watch +- apiGroups: + - ray.io + resources: + - rayclusters/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-rayjob-editor-role +rules: +- apiGroups: + - ray.io + resources: + - rayjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ray.io + resources: + - rayjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-rayjob-viewer-role +rules: +- apiGroups: + - ray.io + resources: + - rayjobs + verbs: + - get + - list + - watch +- apiGroups: + - ray.io + resources: + - rayjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-resourceflavor-editor-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - resourceflavors + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-resourceflavor-viewer-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - resourceflavors + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-tfjob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - tfjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - tfjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-tfjob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - tfjobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - tfjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-workload-editor-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - workloads + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - workloads/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-workload-viewer-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - workloads + verbs: + - get + - list + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - workloads/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-xgboostjob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - xgboostjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - xgboostjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-xgboostjob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - xgboostjobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - xgboostjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-visibility-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: kueue-controller-manager + namespace: kueue-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-leader-election-rolebinding + namespace: kueue-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kueue-leader-election-role +subjects: +- kind: ServiceAccount + name: kueue-controller-manager + namespace: kueue-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kueue-manager-role +subjects: +- kind: ServiceAccount + name: kueue-controller-manager + namespace: kueue-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kueue-proxy-role +subjects: +- kind: ServiceAccount + name: kueue-controller-manager + namespace: kueue-system +--- +apiVersion: v1 +data: + controller_manager_config.yaml: | + apiVersion: config.kueue.x-k8s.io/v1beta1 + kind: Configuration + health: + healthProbeBindAddress: :8081 + metrics: + bindAddress: :8080 + # enableClusterQueueResources: true + webhook: + port: 9443 + leaderElection: + leaderElect: true + resourceName: c1f6bfd2.kueue.x-k8s.io + controller: + groupKindConcurrency: + Job.batch: 5 + Pod: 5 + Workload.kueue.x-k8s.io: 5 + LocalQueue.kueue.x-k8s.io: 1 + Cohort.kueue.x-k8s.io: 1 + ClusterQueue.kueue.x-k8s.io: 1 + ResourceFlavor.kueue.x-k8s.io: 1 + clientConnection: + qps: 50 + burst: 100 + #pprofBindAddress: :8083 + #waitForPodsReady: + # enable: false + # timeout: 5m + # blockAdmission: false + # requeuingStrategy: + # timestamp: Eviction + # backoffLimitCount: null # null indicates infinite requeuing + # backoffBaseSeconds: 60 + # backoffMaxSeconds: 3600 + #manageJobsWithoutQueueName: true + #managedJobsNamespaceSelector: + # matchLabels: + # kueue-managed: "true" + #internalCertManagement: + # enable: false + # webhookServiceName: "" + # webhookSecretName: "" + integrations: + frameworks: + - "batch/job" + - "kubeflow.org/mpijob" + - "ray.io/rayjob" + - "ray.io/raycluster" + - "jobset.x-k8s.io/jobset" + - "kubeflow.org/mxjob" + - "kubeflow.org/paddlejob" + - "kubeflow.org/pytorchjob" + - "kubeflow.org/tfjob" + - "kubeflow.org/xgboostjob" + # - "pod" + # - "deployment" # requires enabling pod integration + # - "statefulset" # requires enabling pod integration + # externalFrameworks: + # - "Foo.v1.example.com" + # podOptions: + # namespaceSelector: + # matchExpressions: + # - key: kubernetes.io/metadata.name + # operator: NotIn + # values: [ kube-system, kueue-system ] + #fairSharing: + # enable: true + # preemptionStrategies: [LessThanOrEqualToFinalShare, LessThanInitialShare] + #resources: + # excludeResourcePrefixes: [] + # transformations: + # - input: nvidia.com/mig-4g.5gb + # strategy: Replace | Retain + # outputs: + # example.com/accelerator-memory: 5Gi + # example.com/accelerator-gpc: 4 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-manager-config + namespace: kueue-system +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-webhook-server-cert + namespace: kueue-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-controller-manager-metrics-service + namespace: kueue-system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + control-plane: controller-manager +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-visibility-server + namespace: kueue-system +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: 8082 + selector: + control-plane: controller-manager +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-webhook-service + namespace: kueue-system +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-controller-manager + namespace: kueue-system +spec: + replicas: 1 + selector: + matchLabels: + control-plane: controller-manager + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + spec: + containers: + - args: + - --config=/controller_manager_config.yaml + - --zap-log-level=2 + - --feature-gates=TopologyAwareScheduling=true + command: + - /manager + image: registry.k8s.io/kueue/kueue:v0.10.1 + imagePullPolicy: Always + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: manager + ports: + - containerPort: 8082 + name: visibility + protocol: TCP + - containerPort: 9443 + name: webhook-server + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + - mountPath: /controller_manager_config.yaml + name: manager-config + subPath: controller_manager_config.yaml + - args: + - --secure-listen-address=0.0.0.0:8443 + - --upstream=http://127.0.0.1:8080/ + - --logtostderr=true + - --v=10 + image: registry.k8s.io/kubebuilder/kube-rbac-proxy:v0.16.0 + name: kube-rbac-proxy + ports: + - containerPort: 8443 + name: https + protocol: TCP + securityContext: + runAsNonRoot: true + serviceAccountName: kueue-controller-manager + terminationGracePeriodSeconds: 10 + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: kueue-webhook-server-cert + - configMap: + name: kueue-manager-config + name: manager-config + tolerations: + - effect: NoSchedule + key: components.gke.io/gke-managed-components + operator: Equal + value: "true" +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: v1beta1.visibility.kueue.x-k8s.io +spec: + group: visibility.kueue.x-k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: kueue-visibility-server + namespace: kueue-system + version: v1beta1 + versionPriority: 100 +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate--v1-pod + failurePolicy: Fail + name: mpod.kb.io + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kueue-system + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-apps-v1-deployment + failurePolicy: Fail + name: mdeployment.kb.io + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kueue-system + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - deployments + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-batch-v1-job + failurePolicy: Fail + name: mjob.kb.io + rules: + - apiGroups: + - batch + apiVersions: + - v1 + operations: + - CREATE + resources: + - jobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-jobset-x-k8s-io-v1alpha2-jobset + failurePolicy: Fail + name: mjobset.kb.io + rules: + - apiGroups: + - jobset.x-k8s.io + apiVersions: + - v1alpha2 + operations: + - CREATE + resources: + - jobsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v1-mxjob + failurePolicy: Fail + name: mmxjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + resources: + - mxjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v1-paddlejob + failurePolicy: Fail + name: mpaddlejob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + resources: + - paddlejobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v1-pytorchjob + failurePolicy: Fail + name: mpytorchjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + resources: + - pytorchjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v1-tfjob + failurePolicy: Fail + name: mtfjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + resources: + - tfjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v1-xgboostjob + failurePolicy: Fail + name: mxgboostjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + resources: + - xgboostjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v2beta1-mpijob + failurePolicy: Fail + name: mmpijob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v2beta1 + operations: + - CREATE + resources: + - mpijobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-ray-io-v1-raycluster + failurePolicy: Fail + name: mraycluster.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + resources: + - rayclusters + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-ray-io-v1-rayjob + failurePolicy: Fail + name: mrayjob.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + resources: + - rayjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-apps-v1-statefulset + failurePolicy: Fail + name: mstatefulset.kb.io + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - statefulsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kueue-x-k8s-io-v1beta1-clusterqueue + failurePolicy: Fail + name: mclusterqueue.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + resources: + - clusterqueues + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kueue-x-k8s-io-v1beta1-resourceflavor + failurePolicy: Fail + name: mresourceflavor.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + resources: + - resourceflavors + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kueue-x-k8s-io-v1beta1-workload + failurePolicy: Fail + name: mworkload.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + resources: + - workloads + sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate--v1-pod + failurePolicy: Fail + name: vpod.kb.io + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kueue-system + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - pods + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-apps-v1-deployment + failurePolicy: Fail + name: vdeployment.kb.io + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kueue-system + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - deployments + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-batch-v1-job + failurePolicy: Fail + name: vjob.kb.io + rules: + - apiGroups: + - batch + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - jobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-jobset-x-k8s-io-v1alpha2-jobset + failurePolicy: Fail + name: vjobset.kb.io + rules: + - apiGroups: + - jobset.x-k8s.io + apiVersions: + - v1alpha2 + operations: + - CREATE + - UPDATE + resources: + - jobsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v1-mxjob + failurePolicy: Fail + name: vmxjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - mxjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v1-paddlejob + failurePolicy: Fail + name: vpaddlejob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - paddlejobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v1-pytorchjob + failurePolicy: Fail + name: vpytorchjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - pytorchjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v1-tfjob + failurePolicy: Fail + name: vtfjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - tfjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v1-xgboostjob + failurePolicy: Fail + name: vxgboostjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - xgboostjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v2beta1-mpijob + failurePolicy: Fail + name: vmpijob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v2beta1 + operations: + - CREATE + - UPDATE + resources: + - mpijobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-ray-io-v1-raycluster + failurePolicy: Fail + name: vraycluster.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - rayclusters + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-ray-io-v1-rayjob + failurePolicy: Fail + name: vrayjob.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - rayjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-apps-v1-statefulset + failurePolicy: Fail + name: vstatefulset.kb.io + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - statefulsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kueue-x-k8s-io-v1beta1-clusterqueue + failurePolicy: Fail + name: vclusterqueue.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - clusterqueues + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kueue-x-k8s-io-v1alpha1-cohort + failurePolicy: Fail + name: vcohort.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - cohorts + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kueue-x-k8s-io-v1beta1-resourceflavor + failurePolicy: Fail + name: vresourceflavor.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - resourceflavors + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kueue-x-k8s-io-v1beta1-workload + failurePolicy: Fail + name: vworkload.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - workloads + - workloads/status + sideEffects: None diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml new file mode 100644 index 0000000000..ca1e208d34 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml @@ -0,0 +1,13466 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-system +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.2 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: admissionchecks.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: AdmissionCheck + listKind: AdmissionCheckList + plural: admissionchecks + singular: admissioncheck + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: AdmissionCheck is the Schema for the admissionchecks API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: AdmissionCheckSpec defines the desired state of AdmissionCheck + properties: + controllerName: + description: |- + controllerName identifies the controller that processes the AdmissionCheck, + not necessarily a Kubernetes Pod or Deployment name. Cannot be empty. + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + parameters: + description: |- + Parameters identifies a configuration with additional parameters for the + check. + properties: + apiGroup: + description: ApiGroup is the group for the resource being referenced. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is the type of the resource being referenced. + maxLength: 63 + pattern: ^(?i)[a-z]([-a-z0-9]*[a-z0-9])?$ + type: string + name: + description: Name is the name of the resource being referenced. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - apiGroup + - kind + - name + type: object + retryDelayMinutes: + default: 15 + description: |- + RetryDelayMinutes specifies how long to keep the workload suspended after + a failed check (after it transitioned to False). When the delay period has passed, the check + state goes to "Unknown". The default is 15 min. + Deprecated: retryDelayMinutes has already been deprecated since v0.8 and will be removed in v1beta2. + format: int64 + type: integer + required: + - controllerName + type: object + status: + description: AdmissionCheckStatus defines the observed state of AdmissionCheck + properties: + conditions: + description: |- + conditions hold the latest available observations of the AdmissionCheck + current state. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + controller-gen.kubebuilder.io/version: v0.17.2 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: clusterqueues.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: ClusterQueue + listKind: ClusterQueueList + plural: clusterqueues + shortNames: + - cq + singular: clusterqueue + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Cohort that this ClusterQueue belongs to + jsonPath: .spec.cohort + name: Cohort + type: string + - description: The queueing strategy used to prioritize workloads + jsonPath: .spec.queueingStrategy + name: Strategy + priority: 1 + type: string + - description: Number of pending workloads + jsonPath: .status.pendingWorkloads + name: Pending Workloads + type: integer + - description: Number of admitted workloads that haven't finished yet + jsonPath: .status.admittedWorkloads + name: Admitted Workloads + priority: 1 + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + description: ClusterQueue is the Schema for the clusterQueue API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ClusterQueueSpec defines the desired state of ClusterQueue + properties: + admissionChecks: + description: |- + admissionChecks lists the AdmissionChecks required by this ClusterQueue. + Cannot be used along with AdmissionCheckStrategy. + items: + type: string + type: array + admissionChecksStrategy: + description: |- + admissionCheckStrategy defines a list of strategies to determine which ResourceFlavors require AdmissionChecks. + This property cannot be used in conjunction with the 'admissionChecks' property. + properties: + admissionChecks: + description: admissionChecks is a list of strategies for AdmissionChecks + items: + description: AdmissionCheckStrategyRule defines rules for a + single AdmissionCheck + properties: + name: + description: name is an AdmissionCheck's name. + type: string + onFlavors: + description: |- + onFlavors is a list of ResourceFlavors' names that this AdmissionCheck should run for. + If empty, the AdmissionCheck will run for all workloads submitted to the ClusterQueue. + items: + description: ResourceFlavorReference is the name of the + ResourceFlavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + type: array + required: + - name + type: object + type: array + type: object + cohort: + description: |- + cohort that this ClusterQueue belongs to. CQs that belong to the + same cohort can borrow unused resources from each other. + + A CQ can be a member of a single borrowing cohort. A workload submitted + to a queue referencing this CQ can borrow quota from any CQ in the cohort. + Only quota for the [resource, flavor] pairs listed in the CQ can be + borrowed. + If empty, this ClusterQueue cannot borrow from any other ClusterQueue and + vice versa. + + A cohort is a name that links CQs together, but it doesn't reference any + object. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + fairSharing: + description: |- + fairSharing defines the properties of the ClusterQueue when + participating in FairSharing. The values are only relevant + if FairSharing is enabled in the Kueue configuration. + properties: + weight: + anyOf: + - type: integer + - type: string + default: 1 + description: |- + weight gives a comparative advantage to this ClusterQueue + or Cohort when competing for unused resources in the + Cohort. The share is based on the dominant resource usage + above nominal quotas for each resource, divided by the + weight. Admission prioritizes scheduling workloads from + ClusterQueues and Cohorts with the lowest share and + preempting workloads from the ClusterQueues and Cohorts + with the highest share. A zero weight implies infinite + share value, meaning that this Node will always be at + disadvantage against other ClusterQueues and Cohorts. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + flavorFungibility: + default: {} + description: |- + flavorFungibility defines whether a workload should try the next flavor + before borrowing or preempting in the flavor being evaluated. + properties: + whenCanBorrow: + default: Borrow + description: |- + whenCanBorrow determines whether a workload should try the next flavor + before borrowing in current flavor. The possible values are: + + - `Borrow` (default): allocate in current flavor if borrowing + is possible. + - `TryNextFlavor`: try next flavor even if the current + flavor has enough resources to borrow. + enum: + - Borrow + - TryNextFlavor + type: string + whenCanPreempt: + default: TryNextFlavor + description: |- + whenCanPreempt determines whether a workload should try the next flavor + before borrowing in current flavor. The possible values are: + + - `Preempt`: allocate in current flavor if it's possible to preempt some workloads. + - `TryNextFlavor` (default): try next flavor even if there are enough + candidates for preemption in the current flavor. + enum: + - Preempt + - TryNextFlavor + type: string + type: object + namespaceSelector: + description: |- + namespaceSelector defines which namespaces are allowed to submit workloads to + this clusterQueue. Beyond this basic support for policy, a policy agent like + Gatekeeper should be used to enforce more advanced policies. + Defaults to null which is a nothing selector (no namespaces eligible). + If set to an empty selector `{}`, then all namespaces are eligible. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + preemption: + default: {} + description: |- + ClusterQueuePreemption contains policies to preempt Workloads from this + ClusterQueue or the ClusterQueue's cohort. + + Preemption may be configured to work in the following scenarios: + + - When a Workload fits within the nominal quota of the ClusterQueue, but + the quota is currently borrowed by other ClusterQueues in the cohort. + We preempt workloads in other ClusterQueues to allow this ClusterQueue to + reclaim its nominal quota. Configured using reclaimWithinCohort. + - When a Workload doesn't fit within the nominal quota of the ClusterQueue + and there are admitted Workloads in the ClusterQueue with lower priority. + Configured using withinClusterQueue. + - When a Workload may fit while both borrowing and preempting + low priority workloads in the Cohort. Configured using borrowWithinCohort. + - When FairSharing is enabled, to maintain fair distribution of + unused resources. See FairSharing documentation. + + The preemption algorithm tries to find a minimal set of Workloads to + preempt to accomomdate the pending Workload, preempting Workloads with + lower priority first. + properties: + borrowWithinCohort: + default: {} + description: |- + BorrowWithinCohort contains configuration which allows to preempt workloads + within cohort while borrowing. It only works with Classical Preemption, + __not__ with Fair Sharing. + properties: + maxPriorityThreshold: + description: |- + maxPriorityThreshold allows to restrict the set of workloads which + might be preempted by a borrowing workload, to only workloads with + priority less than or equal to the specified threshold priority. + When the threshold is not specified, then any workload satisfying the + policy can be preempted by the borrowing workload. + format: int32 + type: integer + policy: + default: Never + description: |- + policy determines the policy for preemption to reclaim quota within cohort while borrowing. + Possible values are: + - `Never` (default): do not allow for preemption, in other + ClusterQueues within the cohort, for a borrowing workload. + - `LowerPriority`: allow preemption, in other ClusterQueues + within the cohort, for a borrowing workload, but only if + the preempted workloads are of lower priority. + enum: + - Never + - LowerPriority + type: string + type: object + reclaimWithinCohort: + default: Never + description: |- + reclaimWithinCohort determines whether a pending Workload can preempt + Workloads from other ClusterQueues in the cohort that are using more than + their nominal quota. The possible values are: + + - `Never` (default): do not preempt Workloads in the cohort. + - `LowerPriority`: **Classic Preemption** if the pending Workload + fits within the nominal quota of its ClusterQueue, only preempt + Workloads in the cohort that have lower priority than the pending + Workload. **Fair Sharing** only preempt Workloads in the cohort that + have lower priority than the pending Workload and that satisfy the + Fair Sharing preemptionStategies. + - `Any`: **Classic Preemption** if the pending Workload fits within + the nominal quota of its ClusterQueue, preempt any Workload in the + cohort, irrespective of priority. **Fair Sharing** preempt Workloads + in the cohort that satisfy the Fair Sharing preemptionStrategies. + enum: + - Never + - LowerPriority + - Any + type: string + withinClusterQueue: + default: Never + description: |- + withinClusterQueue determines whether a pending Workload that doesn't fit + within the nominal quota for its ClusterQueue, can preempt active Workloads in + the ClusterQueue. The possible values are: + + - `Never` (default): do not preempt Workloads in the ClusterQueue. + - `LowerPriority`: only preempt Workloads in the ClusterQueue that have + lower priority than the pending Workload. + - `LowerOrNewerEqualPriority`: only preempt Workloads in the ClusterQueue that + either have a lower priority than the pending workload or equal priority + and are newer than the pending workload. + enum: + - Never + - LowerPriority + - LowerOrNewerEqualPriority + type: string + type: object + x-kubernetes-validations: + - message: reclaimWithinCohort=Never and borrowWithinCohort.Policy!=Never + rule: '!(self.reclaimWithinCohort == ''Never'' && has(self.borrowWithinCohort) + && self.borrowWithinCohort.policy != ''Never'')' + queueingStrategy: + default: BestEffortFIFO + description: |- + QueueingStrategy indicates the queueing strategy of the workloads + across the queues in this ClusterQueue. + Current Supported Strategies: + + - StrictFIFO: workloads are ordered strictly by creation time. + Older workloads that can't be admitted will block admitting newer + workloads even if they fit available quota. + - BestEffortFIFO: workloads are ordered by creation time, + however older workloads that can't be admitted will not block + admitting newer workloads that fit existing quota. + enum: + - StrictFIFO + - BestEffortFIFO + type: string + resourceGroups: + description: |- + resourceGroups describes groups of resources. + Each resource group defines the list of resources and a list of flavors + that provide quotas for these resources. + Each resource and each flavor can only form part of one resource group. + resourceGroups can be up to 16. + items: + properties: + coveredResources: + description: |- + coveredResources is the list of resources covered by the flavors in this + group. + Examples: cpu, memory, vendor.com/gpu. + The list cannot be empty and it can contain up to 16 resources. + items: + description: ResourceName is the name identifying various + resources in a ResourceList. + type: string + maxItems: 16 + minItems: 1 + type: array + flavors: + description: |- + flavors is the list of flavors that provide the resources of this group. + Typically, different flavors represent different hardware models + (e.g., gpu models, cpu architectures) or pricing models (on-demand vs spot + cpus). + Each flavor MUST list all the resources listed for this group in the same + order as the .resources field. + The list cannot be empty and it can contain up to 16 flavors. + items: + properties: + name: + description: |- + name of this flavor. The name should match the .metadata.name of a + ResourceFlavor. If a matching ResourceFlavor does not exist, the + ClusterQueue will have an Active condition set to False. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: |- + resources is the list of quotas for this flavor per resource. + There could be up to 16 resources. + items: + properties: + borrowingLimit: + anyOf: + - type: integer + - type: string + description: |- + borrowingLimit is the maximum amount of quota for the [flavor, resource] + combination that this ClusterQueue is allowed to borrow from the unused + quota of other ClusterQueues in the same cohort. + In total, at a given time, Workloads in a ClusterQueue can consume a + quantity of quota equal to nominalQuota+borrowingLimit, assuming the other + ClusterQueues in the cohort have enough unused quota. + If null, it means that there is no borrowing limit. + If not null, it must be non-negative. + borrowingLimit must be null if spec.cohort is empty. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + lendingLimit: + anyOf: + - type: integer + - type: string + description: |- + lendingLimit is the maximum amount of unused quota for the [flavor, resource] + combination that this ClusterQueue can lend to other ClusterQueues in the same cohort. + In total, at a given time, ClusterQueue reserves for its exclusive use + a quantity of quota equals to nominalQuota - lendingLimit. + If null, it means that there is no lending limit, meaning that + all the nominalQuota can be borrowed by other clusterQueues in the cohort. + If not null, it must be non-negative. + lendingLimit must be null if spec.cohort is empty. + This field is in beta stage and is enabled by default. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name of this resource. + type: string + nominalQuota: + anyOf: + - type: integer + - type: string + description: |- + nominalQuota is the quantity of this resource that is available for + Workloads admitted by this ClusterQueue at a point in time. + The nominalQuota must be non-negative. + nominalQuota should represent the resources in the cluster available for + running jobs (after discounting resources consumed by system components + and pods not managed by kueue). In an autoscaled cluster, nominalQuota + should account for resources that can be provided by a component such as + Kubernetes cluster-autoscaler. + + If the ClusterQueue belongs to a cohort, the sum of the quotas for each + (flavor, resource) combination defines the maximum quantity that can be + allocated by a ClusterQueue in the cohort. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + - nominalQuota + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - coveredResources + - flavors + type: object + x-kubernetes-validations: + - message: flavors must have the same number of resources as the + coveredResources + rule: self.flavors.all(x, size(x.resources) == size(self.coveredResources)) + maxItems: 16 + type: array + x-kubernetes-list-type: atomic + stopPolicy: + default: None + description: |- + stopPolicy - if set to a value different from None, the ClusterQueue is considered Inactive, no new reservation being + made. + + Depending on its value, its associated workloads will: + + - None - Workloads are admitted + - HoldAndDrain - Admitted workloads are evicted and Reserving workloads will cancel the reservation. + - Hold - Admitted workloads will run to completion and Reserving workloads will cancel the reservation. + enum: + - None + - Hold + - HoldAndDrain + type: string + type: object + x-kubernetes-validations: + - message: borrowingLimit must be nil when cohort is empty + rule: '!has(self.cohort) && has(self.resourceGroups) ? self.resourceGroups.all(rg, + rg.flavors.all(f, f.resources.all(r, !has(r.borrowingLimit)))) : true' + status: + description: ClusterQueueStatus defines the observed state of ClusterQueue + properties: + admittedWorkloads: + description: |- + admittedWorkloads is the number of workloads currently admitted to this + clusterQueue and haven't finished yet. + format: int32 + type: integer + conditions: + description: |- + conditions hold the latest available observations of the ClusterQueue + current state. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + fairSharing: + description: fairSharing contains the information about the current + status of Fair Sharing. + properties: + weightedShare: + description: |- + WeightedShare represent the maximum of the ratios of usage + above nominal quota to the lendable resources in the + Cohort, among all the resources provided by the Node, and + divided by the weight. If zero, it means that the usage of + the Node is below the nominal quota. If the Node has a + weight of zero and is borrowing, this will return + 9223372036854775807, the maximum possible share value. + format: int64 + type: integer + required: + - weightedShare + type: object + flavorsReservation: + description: |- + flavorsReservation are the reserved quotas, by flavor, currently in use by the + workloads assigned to this ClusterQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: resources lists the quota usage for the resources + in this flavor. + items: + properties: + borrowed: + anyOf: + - type: integer + - type: string + description: |- + Borrowed is quantity of quota that is borrowed from the cohort. In other + words, it's the used quota that is over the nominalQuota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name of the resource + type: string + total: + anyOf: + - type: integer + - type: string + description: |- + total is the total quantity of used quota, including the amount borrowed + from the cohort. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + flavorsUsage: + description: |- + flavorsUsage are the used quotas, by flavor, currently in use by the + workloads admitted in this ClusterQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: resources lists the quota usage for the resources + in this flavor. + items: + properties: + borrowed: + anyOf: + - type: integer + - type: string + description: |- + Borrowed is quantity of quota that is borrowed from the cohort. In other + words, it's the used quota that is over the nominalQuota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name of the resource + type: string + total: + anyOf: + - type: integer + - type: string + description: |- + total is the total quantity of used quota, including the amount borrowed + from the cohort. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + pendingWorkloads: + description: |- + pendingWorkloads is the number of workloads currently waiting to be + admitted to this clusterQueue. + format: int32 + type: integer + pendingWorkloadsStatus: + description: |- + PendingWorkloadsStatus contains the information exposed about the current + status of the pending workloads in the cluster queue. + Deprecated: This field will be removed on v1beta2, use VisibilityOnDemand + (https://kueue.sigs.k8s.io/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand/) + instead. + properties: + clusterQueuePendingWorkload: + description: Head contains the list of top pending workloads. + items: + description: |- + ClusterQueuePendingWorkload contains the information identifying a pending workload + in the cluster queue. + properties: + name: + description: Name indicates the name of the pending workload. + type: string + namespace: + description: Namespace indicates the name of the pending + workload. + type: string + required: + - name + - namespace + type: object + type: array + x-kubernetes-list-type: atomic + lastChangeTime: + description: LastChangeTime indicates the time of the last change + of the structure. + format: date-time + type: string + required: + - lastChangeTime + type: object + reservingWorkloads: + description: |- + reservingWorkloads is the number of workloads currently reserving quota in this + clusterQueue. + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.2 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: cohorts.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: Cohort + listKind: CohortList + plural: cohorts + singular: cohort + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + Cohort defines the Cohorts API. + + Hierarchical Cohorts (any Cohort which has a parent) are compatible + with Fair Sharing as of v0.11. Using these features together in + V0.9 and V0.10 is unsupported, and results in undefined behavior. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CohortSpec defines the desired state of Cohort + properties: + fairSharing: + description: |- + fairSharing defines the properties of the Cohort when + participating in FairSharing. The values are only relevant + if FairSharing is enabled in the Kueue configuration. + properties: + weight: + anyOf: + - type: integer + - type: string + default: 1 + description: |- + weight gives a comparative advantage to this ClusterQueue + or Cohort when competing for unused resources in the + Cohort. The share is based on the dominant resource usage + above nominal quotas for each resource, divided by the + weight. Admission prioritizes scheduling workloads from + ClusterQueues and Cohorts with the lowest share and + preempting workloads from the ClusterQueues and Cohorts + with the highest share. A zero weight implies infinite + share value, meaning that this Node will always be at + disadvantage against other ClusterQueues and Cohorts. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + parent: + description: |- + Parent references the name of the Cohort's parent, if + any. It satisfies one of three cases: + 1) Unset. This Cohort is the root of its Cohort tree. + 2) References a non-existent Cohort. We use default Cohort (no borrowing/lending limits). + 3) References an existent Cohort. + + If a cycle is created, we disable all members of the + Cohort, including ClusterQueues, until the cycle is + removed. We prevent further admission while the cycle + exists. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resourceGroups: + description: |- + ResourceGroups describes groupings of Resources and + Flavors. Each ResourceGroup defines a list of Resources + and a list of Flavors which provide quotas for these + Resources. Each Resource and each Flavor may only form part + of one ResourceGroup. There may be up to 16 ResourceGroups + within a Cohort. + + BorrowingLimit limits how much members of this Cohort + subtree can borrow from the parent subtree. + + LendingLimit limits how much members of this Cohort subtree + can lend to the parent subtree. + + Borrowing and Lending limits must only be set when the + Cohort has a parent. Otherwise, the Cohort create/update + will be rejected by the webhook. + items: + properties: + coveredResources: + description: |- + coveredResources is the list of resources covered by the flavors in this + group. + Examples: cpu, memory, vendor.com/gpu. + The list cannot be empty and it can contain up to 16 resources. + items: + description: ResourceName is the name identifying various + resources in a ResourceList. + type: string + maxItems: 16 + minItems: 1 + type: array + flavors: + description: |- + flavors is the list of flavors that provide the resources of this group. + Typically, different flavors represent different hardware models + (e.g., gpu models, cpu architectures) or pricing models (on-demand vs spot + cpus). + Each flavor MUST list all the resources listed for this group in the same + order as the .resources field. + The list cannot be empty and it can contain up to 16 flavors. + items: + properties: + name: + description: |- + name of this flavor. The name should match the .metadata.name of a + ResourceFlavor. If a matching ResourceFlavor does not exist, the + ClusterQueue will have an Active condition set to False. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: |- + resources is the list of quotas for this flavor per resource. + There could be up to 16 resources. + items: + properties: + borrowingLimit: + anyOf: + - type: integer + - type: string + description: |- + borrowingLimit is the maximum amount of quota for the [flavor, resource] + combination that this ClusterQueue is allowed to borrow from the unused + quota of other ClusterQueues in the same cohort. + In total, at a given time, Workloads in a ClusterQueue can consume a + quantity of quota equal to nominalQuota+borrowingLimit, assuming the other + ClusterQueues in the cohort have enough unused quota. + If null, it means that there is no borrowing limit. + If not null, it must be non-negative. + borrowingLimit must be null if spec.cohort is empty. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + lendingLimit: + anyOf: + - type: integer + - type: string + description: |- + lendingLimit is the maximum amount of unused quota for the [flavor, resource] + combination that this ClusterQueue can lend to other ClusterQueues in the same cohort. + In total, at a given time, ClusterQueue reserves for its exclusive use + a quantity of quota equals to nominalQuota - lendingLimit. + If null, it means that there is no lending limit, meaning that + all the nominalQuota can be borrowed by other clusterQueues in the cohort. + If not null, it must be non-negative. + lendingLimit must be null if spec.cohort is empty. + This field is in beta stage and is enabled by default. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name of this resource. + type: string + nominalQuota: + anyOf: + - type: integer + - type: string + description: |- + nominalQuota is the quantity of this resource that is available for + Workloads admitted by this ClusterQueue at a point in time. + The nominalQuota must be non-negative. + nominalQuota should represent the resources in the cluster available for + running jobs (after discounting resources consumed by system components + and pods not managed by kueue). In an autoscaled cluster, nominalQuota + should account for resources that can be provided by a component such as + Kubernetes cluster-autoscaler. + + If the ClusterQueue belongs to a cohort, the sum of the quotas for each + (flavor, resource) combination defines the maximum quantity that can be + allocated by a ClusterQueue in the cohort. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + - nominalQuota + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - coveredResources + - flavors + type: object + x-kubernetes-validations: + - message: flavors must have the same number of resources as the + coveredResources + rule: self.flavors.all(x, size(x.resources) == size(self.coveredResources)) + maxItems: 16 + type: array + x-kubernetes-list-type: atomic + type: object + status: + properties: + fairSharing: + description: fairSharing contains the information about the current + status of Fair Sharing. + properties: + weightedShare: + description: |- + WeightedShare represent the maximum of the ratios of usage + above nominal quota to the lendable resources in the + Cohort, among all the resources provided by the Node, and + divided by the weight. If zero, it means that the usage of + the Node is below the nominal quota. If the Node has a + weight of zero and is borrowing, this will return + 9223372036854775807, the maximum possible share value. + format: int64 + type: integer + required: + - weightedShare + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + controller-gen.kubebuilder.io/version: v0.17.2 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: localqueues.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: LocalQueue + listKind: LocalQueueList + plural: localqueues + shortNames: + - queue + - queues + - lq + singular: localqueue + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Backing ClusterQueue + jsonPath: .spec.clusterQueue + name: ClusterQueue + type: string + - description: Number of pending workloads + jsonPath: .status.pendingWorkloads + name: Pending Workloads + type: integer + - description: Number of admitted workloads that haven't finished yet. + jsonPath: .status.admittedWorkloads + name: Admitted Workloads + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + description: LocalQueue is the Schema for the localQueues API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: LocalQueueSpec defines the desired state of LocalQueue + properties: + clusterQueue: + description: clusterQueue is a reference to a clusterQueue that backs + this localQueue. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + stopPolicy: + default: None + description: |- + stopPolicy - if set to a value different from None, the LocalQueue is considered Inactive, + no new reservation being made. + + Depending on its value, its associated workloads will: + + - None - Workloads are admitted + - HoldAndDrain - Admitted workloads are evicted and Reserving workloads will cancel the reservation. + - Hold - Admitted workloads will run to completion and Reserving workloads will cancel the reservation. + enum: + - None + - Hold + - HoldAndDrain + type: string + type: object + status: + description: LocalQueueStatus defines the observed state of LocalQueue + properties: + admittedWorkloads: + description: |- + admittedWorkloads is the number of workloads in this LocalQueue + admitted to a ClusterQueue and that haven't finished yet. + format: int32 + type: integer + conditions: + description: |- + Conditions hold the latest available observations of the LocalQueue + current state. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + flavorUsage: + description: |- + flavorsUsage are the used quotas, by flavor currently in use by the + workloads assigned to this LocalQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: resources lists the quota usage for the resources + in this flavor. + items: + properties: + name: + description: name of the resource. + type: string + total: + anyOf: + - type: integer + - type: string + description: total is the total quantity of used quota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + flavors: + description: flavors lists all currently available ResourceFlavors + in specified ClusterQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + nodeLabels: + additionalProperties: + type: string + description: |- + nodeLabels are labels that associate the ResourceFlavor with Nodes that + have the same labels. + maxProperties: 8 + type: object + x-kubernetes-map-type: atomic + nodeTaints: + description: |- + nodeTaints are taints that the nodes associated with this ResourceFlavor + have. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to + a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint + key. + type: string + required: + - effect + - key + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + resources: + description: resources used in the flavor. + items: + description: ResourceName is the name identifying various + resources in a ResourceList. + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + topology: + description: |- + topology is the topology that associated with this ResourceFlavor. + + This is an alpha field and requires enabling the TopologyAwareScheduling + feature gate. + properties: + levels: + description: levels define the levels of topology. + items: + type: string + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + name: + description: name is the name of the topology. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - levels + - name + type: object + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + flavorsReservation: + description: |- + flavorsReservation are the reserved quotas, by flavor currently in use by the + workloads assigned to this LocalQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: resources lists the quota usage for the resources + in this flavor. + items: + properties: + name: + description: name of the resource. + type: string + total: + anyOf: + - type: integer + - type: string + description: total is the total quantity of used quota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + pendingWorkloads: + description: PendingWorkloads is the number of Workloads in the LocalQueue + not yet admitted to a ClusterQueue + format: int32 + type: integer + reservingWorkloads: + description: |- + reservingWorkloads is the number of workloads in this LocalQueue + reserving quota in a ClusterQueue and that haven't finished yet. + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.2 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: multikueueclusters.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: MultiKueueCluster + listKind: MultiKueueClusterList + plural: multikueueclusters + singular: multikueuecluster + scope: Cluster + versions: + - additionalPrinterColumns: + - description: MultiKueueCluster is connected + jsonPath: .status.conditions[?(@.type=='Active')].status + name: Connected + type: string + - description: Time this workload was created + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: MultiKueueCluster is the Schema for the multikueue API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + kubeConfig: + description: Information how to connect to the cluster. + properties: + location: + description: |- + Location of the KubeConfig. + + If LocationType is Secret then Location is the name of the secret inside the namespace in + which the kueue controller manager is running. The config should be stored in the "kubeconfig" key. + type: string + locationType: + default: Secret + description: Type of the KubeConfig location. + enum: + - Secret + - Path + type: string + required: + - location + - locationType + type: object + required: + - kubeConfig + type: object + status: + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.2 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: multikueueconfigs.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: MultiKueueConfig + listKind: MultiKueueConfigList + plural: multikueueconfigs + singular: multikueueconfig + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: MultiKueueConfig is the Schema for the multikueue API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: MultiKueueConfigSpec defines the desired state of MultiKueueConfig + properties: + clusters: + description: List of MultiKueueClusters names where the workloads + from the ClusterQueue should be distributed. + items: + type: string + maxItems: 10 + minItems: 1 + type: array + x-kubernetes-list-type: set + required: + - clusters + type: object + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.2 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: provisioningrequestconfigs.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: ProvisioningRequestConfig + listKind: ProvisioningRequestConfigList + plural: provisioningrequestconfigs + singular: provisioningrequestconfig + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: ProvisioningRequestConfig is the Schema for the provisioningrequestconfig + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ProvisioningRequestConfigSpec defines the desired state of + ProvisioningRequestConfig + properties: + managedResources: + description: |- + managedResources contains the list of resources managed by the autoscaling. + + If empty, all resources are considered managed. + + If not empty, the ProvisioningRequest will contain only the podsets that are + requesting at least one of them. + + If none of the workloads podsets is requesting at least a managed resource, + the workload is considered ready. + items: + description: ResourceName is the name identifying various resources + in a ResourceList. + type: string + maxItems: 100 + type: array + x-kubernetes-list-type: set + parameters: + additionalProperties: + description: Parameter is limited to 255 characters. + maxLength: 255 + type: string + description: Parameters contains all other parameters classes may + require. + maxProperties: 100 + type: object + provisioningClassName: + description: |- + ProvisioningClassName describes the different modes of provisioning the resources. + Check autoscaling.x-k8s.io ProvisioningRequestSpec.ProvisioningClassName for details. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + retryStrategy: + default: + backoffBaseSeconds: 60 + backoffLimitCount: 3 + backoffMaxSeconds: 1800 + description: |- + retryStrategy defines strategy for retrying ProvisioningRequest. + If null, then the default configuration is applied with the following parameter values: + backoffLimitCount: 3 + backoffBaseSeconds: 60 - 1 min + backoffMaxSeconds: 1800 - 30 mins + + To switch off retry mechanism + set retryStrategy.backoffLimitCount to 0. + properties: + backoffBaseSeconds: + default: 60 + description: |- + BackoffBaseSeconds defines the base for the exponential backoff for + re-queuing an evicted workload. + + Defaults to 60. + format: int32 + type: integer + backoffLimitCount: + default: 3 + description: |- + BackoffLimitCount defines the maximum number of re-queuing retries. + Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). + + Every backoff duration is about "b*2^(n-1)+Rand" where: + - "b" represents the base set by "BackoffBaseSeconds" parameter, + - "n" represents the "workloadStatus.requeueState.count", + - "Rand" represents the random jitter. + During this time, the workload is taken as an inadmissible and + other workloads will have a chance to be admitted. + By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...). + + Defaults to 3. + format: int32 + type: integer + backoffMaxSeconds: + default: 1800 + description: |- + BackoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload. + + Defaults to 1800. + format: int32 + type: integer + type: object + required: + - provisioningClassName + type: object + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + controller-gen.kubebuilder.io/version: v0.17.2 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: resourceflavors.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: ResourceFlavor + listKind: ResourceFlavorList + plural: resourceflavors + shortNames: + - flavor + - flavors + - rf + singular: resourceflavor + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: ResourceFlavor is the Schema for the resourceflavors API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ResourceFlavorSpec defines the desired state of the ResourceFlavor + properties: + nodeLabels: + additionalProperties: + type: string + description: |- + nodeLabels are labels that associate the ResourceFlavor with Nodes that + have the same labels. + When a Workload is admitted, its podsets can only get assigned + ResourceFlavors whose nodeLabels match the nodeSelector and nodeAffinity + fields. + Once a ResourceFlavor is assigned to a podSet, the ResourceFlavor's + nodeLabels should be injected into the pods of the Workload by the + controller that integrates with the Workload object. + + nodeLabels can be up to 8 elements. + maxProperties: 8 + type: object + x-kubernetes-map-type: atomic + nodeTaints: + description: |- + nodeTaints are taints that the nodes associated with this ResourceFlavor + have. + Workloads' podsets must have tolerations for these nodeTaints in order to + get assigned this ResourceFlavor during admission. + Only the 'NoSchedule' and 'NoExecute' taint effects are evaluated, + while 'PreferNoSchedule' is ignored. + + An example of a nodeTaint is + cloud.provider.com/preemptible="true":NoSchedule + + nodeTaints can be up to 8 elements. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: 'supported taint effect values: ''NoSchedule'', ''PreferNoSchedule'', + ''NoExecute''' + rule: self.all(x, x.effect in ['NoSchedule', 'PreferNoSchedule', + 'NoExecute']) + tolerations: + description: |- + tolerations are extra tolerations that will be added to the pods admitted in + the quota associated with this resource flavor. + + An example of a toleration is + cloud.provider.com/preemptible="true":NoSchedule + + tolerations can be up to 8 elements. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: operator must be Exists when 'key' is empty, which means + 'match all values and all keys' + rule: 'self.all(x, !has(x.key) ? x.operator == ''Exists'' : true)' + - message: effect must be 'NoExecute' when 'tolerationSeconds' is + set + rule: 'self.all(x, has(x.tolerationSeconds) ? x.effect == ''NoExecute'' + : true)' + - message: 'supported toleration values: ''Equal''(default), ''Exists''' + rule: self.all(x, !has(x.operator) || x.operator in ['Equal', 'Exists']) + - message: a value must be empty when 'operator' is 'Exists' + rule: 'self.all(x, has(x.operator) && x.operator == ''Exists'' ? + !has(x.value) : true)' + - message: 'supported taint effect values: ''NoSchedule'', ''PreferNoSchedule'', + ''NoExecute''' + rule: self.all(x, !has(x.effect) || x.effect in ['NoSchedule', 'PreferNoSchedule', + 'NoExecute']) + topologyName: + description: |- + topologyName indicates topology for the TAS ResourceFlavor. + When specified, it enables scraping of the topology information from the + nodes matching to the Resource Flavor node labels. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + type: object + x-kubernetes-validations: + - message: at least one nodeLabel is required when topology is set + rule: '!has(self.topologyName) || self.nodeLabels.size() >= 1' + - message: resourceFlavorSpec are immutable when topologyName is set + rule: '!has(oldSelf.topologyName) || self == oldSelf' + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.2 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: topologies.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: Topology + listKind: TopologyList + plural: topologies + singular: topology + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Topology is the Schema for the topology API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: TopologySpec defines the desired state of Topology + properties: + levels: + description: levels define the levels of topology. + items: + description: TopologyLevel defines the desired state of TopologyLevel + properties: + nodeLabel: + description: |- + nodeLabel indicates the name of the node label for a specific topology + level. + + Examples: + - cloud.provider.com/topology-block + - cloud.provider.com/topology-rack + maxLength: 316 + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - nodeLabel + type: object + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + - message: must be unique + rule: size(self.filter(i, size(self.filter(j, j == i)) > 1)) == + 0 + - message: the kubernetes.io/hostname label can only be used at the + lowest level of topology + rule: size(self.filter(i, i.nodeLabel == 'kubernetes.io/hostname')) + == 0 || self[size(self) - 1].nodeLabel == 'kubernetes.io/hostname' + required: + - levels + type: object + required: + - spec + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.2 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: workloadpriorityclasses.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: WorkloadPriorityClass + listKind: WorkloadPriorityClassList + plural: workloadpriorityclasses + singular: workloadpriorityclass + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Value of workloadPriorityClass's Priority + jsonPath: .value + name: Value + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + description: WorkloadPriorityClass is the Schema for the workloadPriorityClass + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + description: + description: |- + description is an arbitrary string that usually provides guidelines on + when this workloadPriorityClass should be used. + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + value: + description: |- + value represents the integer value of this workloadPriorityClass. This is the actual priority that workloads + receive when jobs have the name of this class in their workloadPriorityClass label. + Changing the value of workloadPriorityClass doesn't affect the priority of workloads that were already created. + format: int32 + type: integer + required: + - value + type: object + served: true + storage: true + subresources: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + controller-gen.kubebuilder.io/version: v0.17.2 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: workloads.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: Workload + listKind: WorkloadList + plural: workloads + shortNames: + - wl + singular: workload + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Name of the queue this workload was submitted to + jsonPath: .spec.queueName + name: Queue + type: string + - description: Name of the ClusterQueue where the workload is reserving quota + jsonPath: .status.admission.clusterQueue + name: Reserved in + type: string + - description: Admission status + jsonPath: .status.conditions[?(@.type=='Admitted')].status + name: Admitted + type: string + - description: Workload finished + jsonPath: .status.conditions[?(@.type=='Finished')].status + name: Finished + type: string + - description: Time this workload was created + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Workload is the Schema for the workloads API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: WorkloadSpec defines the desired state of Workload + properties: + active: + default: true + description: |- + Active determines if a workload can be admitted into a queue. + Changing active from true to false will evict any running workloads. + Possible values are: + + - false: indicates that a workload should never be admitted and evicts running workloads + - true: indicates that a workload can be evaluated for admission into it's respective queue. + + Defaults to true + type: boolean + maximumExecutionTimeSeconds: + description: |- + maximumExecutionTimeSeconds if provided, determines the maximum time, in seconds, + the workload can be admitted before it's automatically deactivated. + + If unspecified, no execution time limit is enforced on the Workload. + format: int32 + minimum: 1 + type: integer + podSets: + description: |- + podSets is a list of sets of homogeneous pods, each described by a Pod spec + and a count. + There must be at least one element and at most 8. + podSets cannot be changed. + items: + properties: + count: + default: 1 + description: count is the number of pods for the spec. + format: int32 + minimum: 0 + type: integer + minCount: + description: |- + minCount is the minimum number of pods for the spec acceptable + if the workload supports partial admission. + + If not provided, partial admission for the current PodSet is not + enabled. + + Only one podSet within the workload can use this. + + This is an alpha field and requires enabling PartialAdmission feature gate. + format: int32 + minimum: 1 + type: integer + name: + default: main + description: name is the PodSet name. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + template: + description: |- + template is the Pod template. + + The only allowed fields in template.metadata are labels and annotations. + + If requests are omitted for a container or initContainer, + they default to the limits if they are explicitly specified for the + container or initContainer. + + During admission, the rules in nodeSelector and + nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution that match + the keys in the nodeLabels from the ResourceFlavors considered for this + Workload are used to filter the ResourceFlavors that can be assigned to + this podSet. + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling + rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, + in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules + (e.g. co-locate this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the same + node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates + whether a service account token should be automatically + mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container that you + want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if value + is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command + to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP + GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration + that the container should sleep. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command + to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP + GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration + that the container should sleep. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name of + a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of + a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS resolver + options of a pod. + properties: + name: + description: |- + Name is this DNS resolver option's name. + Required. + type: string + value: + description: Value is this DNS resolver option's + value. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if value + is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed for ephemeral + containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command + to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP + GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration + that the container should sleep. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command + to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP + GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration + that the container should sleep. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral + containers. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + This may only be set for init containers. You cannot set this field on + ephemeral containers. + type: string + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name of + a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of + a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + If this option is set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + of that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container that you + want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if value + is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command + to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP + GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration + that the container should sleep. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command + to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP + GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration + that the container should sleep. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name of + a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of + a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the reference + to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition + in the pod's condition list with matching type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + resources: + description: |- + Resources is the total amount of CPU and Memory resources required by all + containers in the pod. It supports specifying Requests and Limits for + "cpu" and "memory" resource names only. ResourceClaims are not supported. + + This field enables fine-grained control over resource allocation for the + entire pod, allowing resource sharing among containers in a pod. + + This is an alpha field and requires enabling the PodLevelResources feature + gate. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated to a + Pod to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that + applies to the container. + type: string + role: + description: Role is a SELinux role label that + applies to the container. + type: string + type: + description: Type is a SELinux type label that + applies to the container. + type: string + user: + description: User is a SELinux user label that + applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter + to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies how + to spread matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume in a + pod that may be accessed by any container in the + pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + Deprecated: AWSElasticBlockStore is deprecated. All operations for the in-tree + awsElasticBlockStore type are redirected to the ebs.csi.aws.com CSI driver. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: |- + azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. + Deprecated: AzureDisk is deprecated. All operations for the in-tree azureDisk type + are redirected to the disk.csi.azure.com CSI driver. + properties: + cachingMode: + description: 'cachingMode is the Host Caching + mode: None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data + disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk + in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: + multiple blob disks per storage account Dedicated: + single blob disk per storage account Managed: + azure managed data disk (only in managed + availability set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: |- + azureFile represents an Azure File Service mount on the host and bind mount to the pod. + Deprecated: AzureFile is deprecated. All operations for the in-tree azureFile type + are redirected to the file.csi.azure.com CSI driver. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret + that contains Azure Storage Account Name + and Key + type: string + shareName: + description: shareName is the azure share + Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: |- + cephFS represents a Ceph FS mount on the host that shares a pod's lifetime. + Deprecated: CephFS is deprecated and the in-tree cephfs type is no longer supported. + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as the + mounted root, rather than the full Ceph + tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + Deprecated: Cinder is deprecated. All operations for the in-tree cinder type + are redirected to the cinder.csi.openstack.org CSI driver. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap + that should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the + ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) + represents ephemeral storage that is handled + by certain external CSI drivers. + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API + about the pod that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API + volume file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file to + be created. Must not be absolute or + contain the ''..'' path. Must be utf-8 + encoded. The first item of the relative + path must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query + over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding + reference to the PersistentVolume + backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource + that is attached to a kubelet's host machine + and then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target lun + number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target + worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + Deprecated: FlexVolume is deprecated. Consider using a CSIDriver instead. + properties: + driver: + description: driver is the name of the driver + to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field + holds extra command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: |- + flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running. + Deprecated: Flocker is deprecated and the in-tree flocker type is no longer supported. + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the + dataset. This is unique identifier of a + Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + Deprecated: GCEPersistentDisk is deprecated. All operations for the in-tree + gcePersistentDisk type are redirected to the pd.csi.storage.gke.io CSI driver. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + Deprecated: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for + the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether + support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether + support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified + Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun + number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret + for iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: |- + photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine. + Deprecated: PhotonPersistentDisk is deprecated and the in-tree photonPersistentDisk type is no longer supported. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: |- + portworxVolume represents a portworx volume attached and mounted on kubelets host machine. + Deprecated: PortworxVolume is deprecated. All operations for the in-tree portworxVolume type + are redirected to the pxd.portworx.com CSI driver when the CSIMigrationPortworx feature-gate + is on. + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies + a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources + secrets, configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from + the volume root to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about + the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key + to a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether + the ConfigMap or its keys must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information + about the downwardAPI data to project + properties: + items: + description: Items is a list of + DownwardAPIVolume file + items: + description: DownwardAPIVolumeFile + represents information to create + the file containing the pod + field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only + annotations, labels, name, + namespace and uid are supported.' + properties: + apiVersion: + description: Version of + the schema the FieldPath + is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the + field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path + is the relative path name + of the file to be created. + Must not be absolute or + contain the ''..'' path. + Must be utf-8 encoded. The + first item of the relative + path must not start with + ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies + the output format of + the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information about + the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key + to a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify + whether the Secret or its key + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is + information about the serviceAccountToken + data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: |- + quobyte represents a Quobyte mount on the host that shares a pod's lifetime. + Deprecated: Quobyte is deprecated and the in-tree quobyte type is no longer supported. + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references + an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: |- + scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. + Deprecated: ScaleIO is deprecated and the in-tree scaleIO type is no longer supported. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of + the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name + of the ScaleIO Protection Domain for the + configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default + false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage + Pool associated with the protection domain. + type: string + system: + description: system is the name of the storage + system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether + the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: |- + storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. + Deprecated: StorageOS is deprecated and the in-tree storageos type is no longer supported. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: |- + vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine. + Deprecated: VsphereVolume is deprecated. All operations for the in-tree vsphereVolume type + are redirected to the csi.vsphere.vmware.com CSI driver. + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage + Policy Based Management (SPBM) profile ID + associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage + Policy Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies + vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object + topologyRequest: + description: topologyRequest defines the topology request for + the PodSet. + properties: + podIndexLabel: + description: |- + PodIndexLabel indicates the name of the label indexing the pods. + For example, in the context of + - kubernetes job this is: kubernetes.io/job-completion-index + - JobSet: kubernetes.io/job-completion-index (inherited from Job) + - Kubeflow: training.kubeflow.org/replica-index + type: string + preferred: + description: |- + preferred indicates the topology level preferred by the PodSet, as + indicated by the `kueue.x-k8s.io/podset-preferred-topology` PodSet + annotation. + type: string + required: + description: |- + required indicates the topology level required by the PodSet, as + indicated by the `kueue.x-k8s.io/podset-required-topology` PodSet + annotation. + type: string + subGroupCount: + description: |- + SubGroupIndexLabel indicates the count of replicated Jobs (groups) within a PodSet. + For example, in the context of JobSet this value is read from jobset.sigs.k8s.io/replicatedjob-replicas. + format: int32 + type: integer + subGroupIndexLabel: + description: |- + SubGroupIndexLabel indicates the name of the label indexing the instances of replicated Jobs (groups) + within a PodSet. For example, in the context of JobSet this is jobset.sigs.k8s.io/job-index. + type: string + unconstrained: + description: |- + unconstrained indicates that Kueue has the freedom to schedule the PodSet within + the entire available capacity, without constraints on the compactness of the placement. + This is indicated by the `kueue.x-k8s.io/podset-unconstrained-topology` PodSet annotation. + type: boolean + type: object + required: + - count + - template + type: object + x-kubernetes-validations: + - message: minCount should be positive and less or equal to count + rule: 'has(self.minCount) ? self.minCount <= self.count : true' + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + priority: + description: |- + Priority determines the order of access to the resources managed by the + ClusterQueue where the workload is queued. + The priority value is populated from PriorityClassName. + The higher the value, the higher the priority. + If priorityClassName is specified, priority must not be null. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the workload's priority. + "system-node-critical" and "system-cluster-critical" are two special + keywords which indicate the highest priorities with the former being + the highest priority. Any other name must be defined by creating a + PriorityClass object with that name. If not specified, the workload + priority will be default or zero if there is no default. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + priorityClassSource: + default: "" + description: |- + priorityClassSource determines whether the priorityClass field refers to a pod PriorityClass or kueue.x-k8s.io/workloadpriorityclass. + Workload's PriorityClass can accept the name of a pod priorityClass or a workloadPriorityClass. + When using pod PriorityClass, a priorityClassSource field has the scheduling.k8s.io/priorityclass value. + enum: + - kueue.x-k8s.io/workloadpriorityclass + - scheduling.k8s.io/priorityclass + - "" + type: string + queueName: + description: |- + queueName is the name of the LocalQueue the Workload is associated with. + queueName cannot be changed while .status.admission is not null. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - podSets + type: object + x-kubernetes-validations: + - message: priority should not be nil when priorityClassName is set + rule: 'has(self.priorityClassName) ? has(self.priority) : true' + status: + description: WorkloadStatus defines the observed state of Workload + properties: + accumulatedPastExexcutionTimeSeconds: + description: |- + accumulatedPastExexcutionTimeSeconds holds the total time, in seconds, the workload spent + in Admitted state, in the previous `Admit` - `Evict` cycles. + format: int32 + type: integer + admission: + description: |- + admission holds the parameters of the admission of the workload by a + ClusterQueue. admission can be set back to null, but its fields cannot be + changed once set. + properties: + clusterQueue: + description: clusterQueue is the name of the ClusterQueue that + admitted this workload. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + podSetAssignments: + description: PodSetAssignments hold the admission results for + each of the .spec.podSets entries. + items: + properties: + count: + description: |- + count is the number of pods taken into account at admission time. + This field will not change in case of quota reclaim. + Value could be missing for Workloads created before this field was added, + in that case spec.podSets[*].count value will be used. + format: int32 + minimum: 0 + type: integer + flavors: + additionalProperties: + description: ResourceFlavorReference is the name of the + ResourceFlavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + description: Flavors are the flavors assigned to the workload + for each resource. + type: object + name: + default: main + description: Name is the name of the podSet. It should match + one of the names in .spec.podSets. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + resourceUsage: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + resourceUsage keeps track of the total resources all the pods in the podset need to run. + + Beside what is provided in podSet's specs, this calculation takes into account + the LimitRange defaults and RuntimeClass overheads at the moment of admission. + This field will not change in case of quota reclaim. + type: object + topologyAssignment: + description: |- + topologyAssignment indicates the topology assignment divided into + topology domains corresponding to the lowest level of the topology. + The assignment specifies the number of Pods to be scheduled per topology + domain and specifies the node selectors for each topology domain, in the + following way: the node selector keys are specified by the levels field + (same for all domains), and the corresponding node selector value is + specified by the domains.values subfield. If the TopologySpec.Levels field contains + "kubernetes.io/hostname" label, topologyAssignment will contain data only for + this label, and omit higher levels in the topology + + Example: + + topologyAssignment: + levels: + - cloud.provider.com/topology-block + - cloud.provider.com/topology-rack + domains: + - values: [block-1, rack-1] + count: 4 + - values: [block-1, rack-2] + count: 2 + + Here: + - 4 Pods are to be scheduled on nodes matching the node selector: + cloud.provider.com/topology-block: block-1 + cloud.provider.com/topology-rack: rack-1 + - 2 Pods are to be scheduled on nodes matching the node selector: + cloud.provider.com/topology-block: block-1 + cloud.provider.com/topology-rack: rack-2 + + Example: + Below there is an equivalent of the above example assuming, Topology + object defines kubernetes.io/hostname as the lowest level in topology. + Hence we omit higher level of topologies, since the hostname label + is sufficient to explicitly identify a proper node. + + topologyAssignment: + levels: + - kubernetes.io/hostname + domains: + - values: [hostname-1] + count: 4 + - values: [hostname-2] + count: 2 + properties: + domains: + description: |- + domains is a list of topology assignments split by topology domains at + the lowest level of the topology. + items: + properties: + count: + description: |- + count indicates the number of Pods to be scheduled in the topology + domain indicated by the values field. + format: int32 + minimum: 1 + type: integer + values: + description: |- + values is an ordered list of node selector values describing a topology + domain. The values correspond to the consecutive topology levels, from + the highest to the lowest. + items: + type: string + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + required: + - count + - values + type: object + type: array + levels: + description: |- + levels is an ordered list of keys denoting the levels of the assigned + topology (i.e. node label keys), from the highest to the lowest level of + the topology. + items: + type: string + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + required: + - domains + - levels + type: object + required: + - name + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - clusterQueue + - podSetAssignments + type: object + admissionChecks: + description: admissionChecks list all the admission checks required + by the workload and the current status + items: + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + name: + description: name identifies the admission check. + maxLength: 316 + type: string + podSetUpdates: + items: + description: |- + PodSetUpdate contains a list of pod set modifications suggested by AdmissionChecks. + The modifications should be additive only - modifications of already existing keys + or having the same key provided by multiple AdmissionChecks is not allowed and will + result in failure during workload admission. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + name: + description: Name of the PodSet to modify. Should match + to one of the Workload's PodSets. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + nodeSelector: + additionalProperties: + type: string + type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + maxItems: 8 + type: array + x-kubernetes-validations: + - message: operator must be Exists when 'key' is empty, + which means 'match all values and all keys' + rule: 'self.all(x, !has(x.key) ? x.operator == ''Exists'' + : true)' + - message: effect must be 'NoExecute' when 'tolerationSeconds' + is set + rule: 'self.all(x, has(x.tolerationSeconds) ? x.effect + == ''NoExecute'' : true)' + - message: 'supported toleration values: ''Equal''(default), + ''Exists''' + rule: self.all(x, !has(x.operator) || x.operator in + ['Equal', 'Exists']) + - message: a value must be empty when 'operator' is 'Exists' + rule: 'self.all(x, has(x.operator) && x.operator == + ''Exists'' ? !has(x.value) : true)' + - message: 'supported taint effect values: ''NoSchedule'', + ''PreferNoSchedule'', ''NoExecute''' + rule: self.all(x, !has(x.effect) || x.effect in ['NoSchedule', + 'PreferNoSchedule', 'NoExecute']) + required: + - name + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + state: + description: state of the admissionCheck, one of Pending, Ready, + Retry, Rejected + enum: + - Pending + - Ready + - Retry + - Rejected + type: string + required: + - lastTransitionTime + - message + - name + - state + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + conditions: + description: |- + conditions hold the latest available observations of the Workload + current state. + + The type of the condition could be: + + - Admitted: the Workload was admitted through a ClusterQueue. + - Finished: the associated workload finished running (failed or succeeded). + - PodsReady: at least `.spec.podSets[*].count` Pods are ready or have + succeeded. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + reclaimablePods: + description: |- + reclaimablePods keeps track of the number pods within a podset for which + the resource reservation is no longer needed. + items: + properties: + count: + description: count is the number of pods for which the requested + resources are no longer needed. + format: int32 + minimum: 0 + type: integer + name: + description: name is the PodSet name. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - count + - name + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + requeueState: + description: |- + requeueState holds the re-queue state + when a workload meets Eviction with PodsReadyTimeout reason. + properties: + count: + description: |- + count records the number of times a workload has been re-queued + When a deactivated (`.spec.activate`=`false`) workload is reactivated (`.spec.activate`=`true`), + this count would be reset to null. + format: int32 + minimum: 0 + type: integer + requeueAt: + description: |- + requeueAt records the time when a workload will be re-queued. + When a deactivated (`.spec.activate`=`false`) workload is reactivated (`.spec.activate`=`true`), + this time would be reset to null. + format: date-time + type: string + type: object + resourceRequests: + description: |- + resourceRequests provides a detailed view of the resources that were + requested by a non-admitted workload when it was considered for admission. + If admission is non-null, resourceRequests will be empty because + admission.resourceUsage contains the detailed information. + items: + properties: + name: + default: main + description: name is the name of the podSet. It should match + one of the names in .spec.podSets. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + resources is the total resources all the pods in the podset need to run. + + Beside what is provided in podSet's specs, this value also takes into account + the LimitRange defaults and RuntimeClass overheads at the moment of consideration + and the application of resource.excludeResourcePrefixes and resource.transformations. + type: object + required: + - name + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + type: object + x-kubernetes-validations: + - message: podSetAssignments must have the same number of podSets as the spec + rule: 'has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, + c.type == ''QuotaReserved'' && c.status == ''True'') && has(self.status.admission) + ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) + : true' + - message: field is immutable + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, + c.type == ''QuotaReserved'' && c.status == ''True'')) ? (oldSelf.spec.priorityClassSource + == self.spec.priorityClassSource) : true' + - message: field is immutable + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, + c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassName) + && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName + == self.spec.priorityClassName) : true' + - message: field is immutable + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, + c.type == ''QuotaReserved'' && c.status == ''True'')) && (has(self.status) + && has(self.status.conditions) && self.status.conditions.exists(c, c.type + == ''QuotaReserved'' && c.status == ''True'')) && has(oldSelf.spec.queueName) + && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName + : true' + - message: maximumExecutionTimeSeconds is immutable while admitted + rule: ((has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, + c.type == 'Admitted' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) + && self.status.conditions.exists(c, c.type == 'Admitted' && c.status == + 'True')))?((has(oldSelf.spec.maximumExecutionTimeSeconds)?oldSelf.spec.maximumExecutionTimeSeconds:0) + == (has(self.spec.maximumExecutionTimeSeconds)?self.spec.maximumExecutionTimeSeconds:0)):true + served: true + storage: true + subresources: + status: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-controller-manager + namespace: kueue-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-leader-election-role + namespace: kueue-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.kueue.x-k8s.io/batch-admin: "true" +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-batch-admin-role +--- +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.kueue.x-k8s.io/batch-user: "true" +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-batch-user-role +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-clusterqueue-editor-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-clusterqueue-viewer-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues + verbs: + - get + - list + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-job-editor-role +rules: +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch + resources: + - jobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-job-viewer-role +rules: +- apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch +- apiGroups: + - batch + resources: + - jobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-jobset-editor-role +rules: +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-jobset-viewer-role +rules: +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets + verbs: + - get + - list + - watch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-localqueue-editor-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - localqueues + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - localqueues/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-localqueue-viewer-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - localqueues + verbs: + - get + - list + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - localqueues/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-manager-role +rules: +- apiGroups: + - batch + resources: + - cronjobs + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - watch +- apiGroups: + - "" + resources: + - limitranges + - namespaces + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods/finalizers + verbs: + - get + - update +- apiGroups: + - "" + resources: + - pods/status + verbs: + - get + - patch +- apiGroups: + - "" + resources: + - podtemplates + verbs: + - create + - delete + - get + - list + - update + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - get + - list + - update + - watch +- apiGroups: + - apps + resources: + - deployments + - replicasets + - statefulsets + verbs: + - get + - list + - watch +- apiGroups: + - autoscaling.x-k8s.io + resources: + - provisioningrequests + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - autoscaling.x-k8s.io + resources: + - provisioningrequests/status + verbs: + - get +- apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - batch + resources: + - jobs/finalizers + - jobs/status + verbs: + - get + - patch + - update +- apiGroups: + - flowcontrol.apiserver.k8s.io + resources: + - flowschemas + - prioritylevelconfigurations + verbs: + - list + - watch +- apiGroups: + - flowcontrol.apiserver.k8s.io + resources: + - flowschemas/status + verbs: + - patch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/finalizers + verbs: + - get + - update +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/status + verbs: + - get + - patch + - update +- apiGroups: + - kubeflow.org + resources: + - mpijobs + - paddlejobs + - pytorchjobs + - tfjobs + - xgboostjobs + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - mpijobs/finalizers + - paddlejobs/finalizers + - pytorchjobs/finalizers + - tfjobs/finalizers + - xgboostjobs/finalizers + verbs: + - get + - update +- apiGroups: + - kubeflow.org + resources: + - mpijobs/status + - paddlejobs/status + - pytorchjobs/status + - tfjobs/status + - xgboostjobs/status + verbs: + - get + - patch + - update +- apiGroups: + - kueue.x-k8s.io + resources: + - admissionchecks + - clusterqueues + - cohorts + - localqueues + - workloads + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - admissionchecks/finalizers + - clusterqueues/finalizers + - localqueues/finalizers + - resourceflavors/finalizers + - topologies/finalizers + - workloads/finalizers + verbs: + - update +- apiGroups: + - kueue.x-k8s.io + resources: + - admissionchecks/status + - clusterqueues/status + - cohorts/status + - localqueues/status + - multikueueclusters/status + - workloads/status + verbs: + - get + - patch + - update +- apiGroups: + - kueue.x-k8s.io + resources: + - multikueueclusters + - multikueueconfigs + - provisioningrequestconfigs + - workloadpriorityclasses + verbs: + - get + - list + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - resourceflavors + verbs: + - delete + - get + - list + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - topologies + verbs: + - get + - list + - update + - watch +- apiGroups: + - leaderworkerset.x-k8s.io + resources: + - leaderworkersets + verbs: + - get + - list + - watch +- apiGroups: + - node.k8s.io + resources: + - runtimeclasses + verbs: + - get + - list + - watch +- apiGroups: + - ray.io + resources: + - rayclusters + - rayjobs + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - ray.io + resources: + - rayclusters/finalizers + - rayjobs/finalizers + verbs: + - get + - update +- apiGroups: + - ray.io + resources: + - rayclusters/status + - rayjobs/status + verbs: + - get + - patch + - update +- apiGroups: + - scheduling.k8s.io + resources: + - priorityclasses + verbs: + - get + - list + - watch +- apiGroups: + - workload.codeflare.dev + resources: + - appwrappers + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - workload.codeflare.dev + resources: + - appwrappers/finalizers + verbs: + - get + - update +- apiGroups: + - workload.codeflare.dev + resources: + - appwrappers/status + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-mpijob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - mpijobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - mpijobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-mpijob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - mpijobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - mpijobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-paddlejob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - paddlejobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - paddlejobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-paddlejob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - paddlejobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - paddlejobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-pending-workloads-cq-viewer-role +rules: +- apiGroups: + - visibility.kueue.x-k8s.io + resources: + - clusterqueues/pendingworkloads + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-pending-workloads-lq-viewer-role +rules: +- apiGroups: + - visibility.kueue.x-k8s.io + resources: + - localqueues/pendingworkloads + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-pytorchjob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-pytorchjob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-raycluster-editor-role +rules: +- apiGroups: + - ray.io + resources: + - rayclusters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ray.io + resources: + - rayclusters/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-raycluster-viewer-role +rules: +- apiGroups: + - ray.io + resources: + - rayclusters + verbs: + - get + - list + - watch +- apiGroups: + - ray.io + resources: + - rayclusters/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-rayjob-editor-role +rules: +- apiGroups: + - ray.io + resources: + - rayjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ray.io + resources: + - rayjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-rayjob-viewer-role +rules: +- apiGroups: + - ray.io + resources: + - rayjobs + verbs: + - get + - list + - watch +- apiGroups: + - ray.io + resources: + - rayjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-resourceflavor-editor-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - resourceflavors + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-resourceflavor-viewer-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - resourceflavors + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-tfjob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - tfjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - tfjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-tfjob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - tfjobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - tfjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-topology-editor-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - topology + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-topology-viewer-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - topology + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-workload-editor-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - workloads + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - workloads/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-workload-viewer-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - workloads + verbs: + - get + - list + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - workloads/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-xgboostjob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - xgboostjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - xgboostjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-xgboostjob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - xgboostjobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - xgboostjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-visibility-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: kueue-controller-manager + namespace: kueue-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-leader-election-rolebinding + namespace: kueue-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kueue-leader-election-role +subjects: +- kind: ServiceAccount + name: kueue-controller-manager + namespace: kueue-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kueue-manager-role +subjects: +- kind: ServiceAccount + name: kueue-controller-manager + namespace: kueue-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kueue-metrics-auth-role +subjects: +- kind: ServiceAccount + name: kueue-controller-manager + namespace: kueue-system +--- +apiVersion: v1 +data: + controller_manager_config.yaml: | + apiVersion: config.kueue.x-k8s.io/v1beta1 + kind: Configuration + health: + healthProbeBindAddress: :8081 + metrics: + bindAddress: :8443 + # enableClusterQueueResources: true + webhook: + port: 9443 + leaderElection: + leaderElect: true + resourceName: c1f6bfd2.kueue.x-k8s.io + controller: + groupKindConcurrency: + Job.batch: 5 + Pod: 5 + Workload.kueue.x-k8s.io: 5 + LocalQueue.kueue.x-k8s.io: 1 + Cohort.kueue.x-k8s.io: 1 + ClusterQueue.kueue.x-k8s.io: 1 + ResourceFlavor.kueue.x-k8s.io: 1 + clientConnection: + qps: 50 + burst: 100 + #pprofBindAddress: :8083 + #waitForPodsReady: + # enable: false + # timeout: 5m + # recoveryTimeout: 3m + # blockAdmission: false + # requeuingStrategy: + # timestamp: Eviction + # backoffLimitCount: null # null indicates infinite requeuing + # backoffBaseSeconds: 60 + # backoffMaxSeconds: 3600 + #manageJobsWithoutQueueName: true + #managedJobsNamespaceSelector: + # matchExpressions: + # - key: kubernetes.io/metadata.name + # operator: NotIn + # values: [ kube-system, kueue-system ] + #internalCertManagement: + # enable: false + # webhookServiceName: "" + # webhookSecretName: "" + integrations: + frameworks: + - "batch/job" + - "kubeflow.org/mpijob" + - "ray.io/rayjob" + - "ray.io/raycluster" + - "jobset.x-k8s.io/jobset" + - "kubeflow.org/paddlejob" + - "kubeflow.org/pytorchjob" + - "kubeflow.org/tfjob" + - "kubeflow.org/xgboostjob" + - "workload.codeflare.dev/appwrapper" + # - "pod" + # - "deployment" # requires enabling pod integration + # - "statefulset" # requires enabling pod integration + # - "leaderworkerset.x-k8s.io/leaderworkerset" # requires enabling pod integration + # externalFrameworks: + # - "Foo.v1.example.com" + #fairSharing: + # enable: true + # preemptionStrategies: [LessThanOrEqualToFinalShare, LessThanInitialShare] + #resources: + # excludeResourcePrefixes: [] + # transformations: + # - input: nvidia.com/mig-4g.5gb + # strategy: Replace | Retain + # outputs: + # example.com/accelerator-memory: 5Gi + # example.com/accelerator-gpc: 4 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-manager-config + namespace: kueue-system +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-webhook-server-cert + namespace: kueue-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-controller-manager-metrics-service + namespace: kueue-system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + control-plane: controller-manager +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-visibility-server + namespace: kueue-system +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: 8082 + selector: + control-plane: controller-manager +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-webhook-service + namespace: kueue-system +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-controller-manager + namespace: kueue-system +spec: + replicas: 1 + selector: + matchLabels: + control-plane: controller-manager + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + spec: + containers: + - args: + - --config=/controller_manager_config.yaml + - --zap-log-level=2 + - --feature-gates=TopologyAwareScheduling=true + command: + - /manager + image: registry.k8s.io/kueue/kueue:v0.11.4 + imagePullPolicy: Always + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: manager + ports: + - containerPort: 8443 + name: metrics + protocol: TCP + - containerPort: 8082 + name: visibility + protocol: TCP + - containerPort: 9443 + name: webhook-server + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: "2" + memory: 512Mi + requests: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + - mountPath: /controller_manager_config.yaml + name: manager-config + subPath: controller_manager_config.yaml + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + serviceAccountName: kueue-controller-manager + terminationGracePeriodSeconds: 10 + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: kueue-webhook-server-cert + - configMap: + name: kueue-manager-config + name: manager-config + tolerations: + - effect: NoSchedule + key: components.gke.io/gke-managed-components + operator: Equal + value: "true" +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: v1beta1.visibility.kueue.x-k8s.io +spec: + group: visibility.kueue.x-k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: kueue-visibility-server + namespace: kueue-system + version: v1beta1 + versionPriority: 100 +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate--v1-pod + failurePolicy: Fail + name: mpod.kb.io + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kueue-system + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-apps-v1-deployment + failurePolicy: Fail + name: mdeployment.kb.io + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kueue-system + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - deployments + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-apps-v1-statefulset + failurePolicy: Fail + name: mstatefulset.kb.io + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kueue-system + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - statefulsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-workload-codeflare-dev-v1beta2-appwrapper + failurePolicy: Fail + name: mappwrapper.kb.io + rules: + - apiGroups: + - workload.codeflare.dev + apiVersions: + - v1beta2 + operations: + - CREATE + resources: + - appwrappers + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-batch-v1-job + failurePolicy: Fail + name: mjob.kb.io + rules: + - apiGroups: + - batch + apiVersions: + - v1 + operations: + - CREATE + resources: + - jobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-jobset-x-k8s-io-v1alpha2-jobset + failurePolicy: Fail + name: mjobset.kb.io + rules: + - apiGroups: + - jobset.x-k8s.io + apiVersions: + - v1alpha2 + operations: + - CREATE + resources: + - jobsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v1-paddlejob + failurePolicy: Fail + name: mpaddlejob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + resources: + - paddlejobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v1-pytorchjob + failurePolicy: Fail + name: mpytorchjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + resources: + - pytorchjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v1-tfjob + failurePolicy: Fail + name: mtfjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + resources: + - tfjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v1-xgboostjob + failurePolicy: Fail + name: mxgboostjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + resources: + - xgboostjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-leaderworkerset-x-k8s-io-v1-leaderworkerset + failurePolicy: Fail + name: mleaderworkerset.kb.io + rules: + - apiGroups: + - leaderworkerset.x-k8s.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - leaderworkersets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v2beta1-mpijob + failurePolicy: Fail + name: mmpijob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v2beta1 + operations: + - CREATE + resources: + - mpijobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-ray-io-v1-raycluster + failurePolicy: Fail + name: mraycluster.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + resources: + - rayclusters + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-ray-io-v1-rayjob + failurePolicy: Fail + name: mrayjob.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + resources: + - rayjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kueue-x-k8s-io-v1beta1-clusterqueue + failurePolicy: Fail + name: mclusterqueue.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + resources: + - clusterqueues + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kueue-x-k8s-io-v1beta1-resourceflavor + failurePolicy: Fail + name: mresourceflavor.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + resources: + - resourceflavors + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kueue-x-k8s-io-v1beta1-workload + failurePolicy: Fail + name: mworkload.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + resources: + - workloads + sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate--v1-pod + failurePolicy: Fail + name: vpod.kb.io + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kueue-system + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - pods + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-apps-v1-deployment + failurePolicy: Fail + name: vdeployment.kb.io + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kueue-system + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - deployments + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-apps-v1-statefulset + failurePolicy: Fail + name: vstatefulset.kb.io + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kueue-system + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - statefulsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-workload-codeflare-dev-v1beta2-appwrapper + failurePolicy: Fail + name: vappwrapper.kb.io + rules: + - apiGroups: + - workload.codeflare.dev + apiVersions: + - v1beta2 + operations: + - CREATE + - UPDATE + resources: + - appwrappers + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-batch-v1-job + failurePolicy: Fail + name: vjob.kb.io + rules: + - apiGroups: + - batch + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - jobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-jobset-x-k8s-io-v1alpha2-jobset + failurePolicy: Fail + name: vjobset.kb.io + rules: + - apiGroups: + - jobset.x-k8s.io + apiVersions: + - v1alpha2 + operations: + - CREATE + - UPDATE + resources: + - jobsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v1-paddlejob + failurePolicy: Fail + name: vpaddlejob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - paddlejobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v1-pytorchjob + failurePolicy: Fail + name: vpytorchjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - pytorchjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v1-tfjob + failurePolicy: Fail + name: vtfjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - tfjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v1-xgboostjob + failurePolicy: Fail + name: vxgboostjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - xgboostjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-leaderworkerset-x-k8s-io-v1-leaderworkerset + failurePolicy: Fail + name: vleaderworkerset.kb.io + rules: + - apiGroups: + - leaderworkerset.x-k8s.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - leaderworkersets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v2beta1-mpijob + failurePolicy: Fail + name: vmpijob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v2beta1 + operations: + - CREATE + - UPDATE + resources: + - mpijobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-ray-io-v1-raycluster + failurePolicy: Fail + name: vraycluster.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - rayclusters + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-ray-io-v1-rayjob + failurePolicy: Fail + name: vrayjob.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - rayjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kueue-x-k8s-io-v1beta1-clusterqueue + failurePolicy: Fail + name: vclusterqueue.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - clusterqueues + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kueue-x-k8s-io-v1alpha1-cohort + failurePolicy: Fail + name: vcohort.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - cohorts + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kueue-x-k8s-io-v1beta1-resourceflavor + failurePolicy: Fail + name: vresourceflavor.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - resourceflavors + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kueue-x-k8s-io-v1beta1-workload + failurePolicy: Fail + name: vworkload.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - workloads + - workloads/status + sideEffects: None diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.12.2.yaml b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.12.2.yaml new file mode 100644 index 0000000000..3c3f1a1783 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.12.2.yaml @@ -0,0 +1,13851 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-system +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: admissionchecks.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: AdmissionCheck + listKind: AdmissionCheckList + plural: admissionchecks + singular: admissioncheck + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: AdmissionCheck is the Schema for the admissionchecks API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: AdmissionCheckSpec defines the desired state of AdmissionCheck + properties: + controllerName: + description: |- + controllerName identifies the controller that processes the AdmissionCheck, + not necessarily a Kubernetes Pod or Deployment name. Cannot be empty. + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + parameters: + description: |- + Parameters identifies a configuration with additional parameters for the + check. + properties: + apiGroup: + description: ApiGroup is the group for the resource being referenced. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is the type of the resource being referenced. + maxLength: 63 + pattern: ^(?i)[a-z]([-a-z0-9]*[a-z0-9])?$ + type: string + name: + description: Name is the name of the resource being referenced. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - apiGroup + - kind + - name + type: object + retryDelayMinutes: + default: 15 + description: |- + RetryDelayMinutes specifies how long to keep the workload suspended after + a failed check (after it transitioned to False). When the delay period has passed, the check + state goes to "Unknown". The default is 15 min. + Deprecated: retryDelayMinutes has already been deprecated since v0.8 and will be removed in v1beta2. + format: int64 + type: integer + required: + - controllerName + type: object + status: + description: AdmissionCheckStatus defines the observed state of AdmissionCheck + properties: + conditions: + description: |- + conditions hold the latest available observations of the AdmissionCheck + current state. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: clusterqueues.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: ClusterQueue + listKind: ClusterQueueList + plural: clusterqueues + shortNames: + - cq + singular: clusterqueue + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Cohort that this ClusterQueue belongs to + jsonPath: .spec.cohort + name: Cohort + type: string + - description: The queueing strategy used to prioritize workloads + jsonPath: .spec.queueingStrategy + name: Strategy + priority: 1 + type: string + - description: Number of pending workloads + jsonPath: .status.pendingWorkloads + name: Pending Workloads + type: integer + - description: Number of admitted workloads that haven't finished yet + jsonPath: .status.admittedWorkloads + name: Admitted Workloads + priority: 1 + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + description: ClusterQueue is the Schema for the clusterQueue API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ClusterQueueSpec defines the desired state of ClusterQueue + properties: + admissionChecks: + description: |- + admissionChecks lists the AdmissionChecks required by this ClusterQueue. + Cannot be used along with AdmissionCheckStrategy. + items: + description: AdmissionCheckReference is the name of an AdmissionCheck. + maxLength: 316 + type: string + type: array + admissionChecksStrategy: + description: |- + admissionCheckStrategy defines a list of strategies to determine which ResourceFlavors require AdmissionChecks. + This property cannot be used in conjunction with the 'admissionChecks' property. + properties: + admissionChecks: + description: admissionChecks is a list of strategies for AdmissionChecks + items: + description: AdmissionCheckStrategyRule defines rules for a + single AdmissionCheck + properties: + name: + description: name is an AdmissionCheck's name. + maxLength: 316 + type: string + onFlavors: + description: |- + onFlavors is a list of ResourceFlavors' names that this AdmissionCheck should run for. + If empty, the AdmissionCheck will run for all workloads submitted to the ClusterQueue. + items: + description: ResourceFlavorReference is the name of the + ResourceFlavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + type: array + required: + - name + type: object + type: array + type: object + admissionScope: + description: admissionScope indicates whether ClusterQueue uses the + Admission Fair Sharing + properties: + admissionMode: + description: |- + AdmissionMode indicates which mode for AdmissionFairSharing should be used + in the AdmissionScope. Possible values are: + - UsageBasedAdmissionFairSharing + - NoAdmissionFairSharing + type: string + required: + - admissionMode + type: object + cohort: + description: |- + cohort that this ClusterQueue belongs to. CQs that belong to the + same cohort can borrow unused resources from each other. + + A CQ can be a member of a single borrowing cohort. A workload submitted + to a queue referencing this CQ can borrow quota from any CQ in the cohort. + Only quota for the [resource, flavor] pairs listed in the CQ can be + borrowed. + If empty, this ClusterQueue cannot borrow from any other ClusterQueue and + vice versa. + + A cohort is a name that links CQs together, but it doesn't reference any + object. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + fairSharing: + description: |- + fairSharing defines the properties of the ClusterQueue when + participating in FairSharing. The values are only relevant + if FairSharing is enabled in the Kueue configuration. + properties: + weight: + anyOf: + - type: integer + - type: string + default: 1 + description: |- + weight gives a comparative advantage to this ClusterQueue + or Cohort when competing for unused resources in the + Cohort. The share is based on the dominant resource usage + above nominal quotas for each resource, divided by the + weight. Admission prioritizes scheduling workloads from + ClusterQueues and Cohorts with the lowest share and + preempting workloads from the ClusterQueues and Cohorts + with the highest share. A zero weight implies infinite + share value, meaning that this Node will always be at + disadvantage against other ClusterQueues and Cohorts. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + flavorFungibility: + default: {} + description: |- + flavorFungibility defines whether a workload should try the next flavor + before borrowing or preempting in the flavor being evaluated. + properties: + whenCanBorrow: + default: Borrow + description: |- + whenCanBorrow determines whether a workload should try the next flavor + before borrowing in current flavor. The possible values are: + + - `Borrow` (default): allocate in current flavor if borrowing + is possible. + - `TryNextFlavor`: try next flavor even if the current + flavor has enough resources to borrow. + enum: + - Borrow + - TryNextFlavor + type: string + whenCanPreempt: + default: TryNextFlavor + description: |- + whenCanPreempt determines whether a workload should try the next flavor + before borrowing in current flavor. The possible values are: + + - `Preempt`: allocate in current flavor if it's possible to preempt some workloads. + - `TryNextFlavor` (default): try next flavor even if there are enough + candidates for preemption in the current flavor. + enum: + - Preempt + - TryNextFlavor + type: string + type: object + namespaceSelector: + description: |- + namespaceSelector defines which namespaces are allowed to submit workloads to + this clusterQueue. Beyond this basic support for policy, a policy agent like + Gatekeeper should be used to enforce more advanced policies. + Defaults to null which is a nothing selector (no namespaces eligible). + If set to an empty selector `{}`, then all namespaces are eligible. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + preemption: + default: {} + description: |- + ClusterQueuePreemption contains policies to preempt Workloads from this + ClusterQueue or the ClusterQueue's cohort. + + Preemption may be configured to work in the following scenarios: + + - When a Workload fits within the nominal quota of the ClusterQueue, but + the quota is currently borrowed by other ClusterQueues in the cohort. + We preempt workloads in other ClusterQueues to allow this ClusterQueue to + reclaim its nominal quota. Configured using reclaimWithinCohort. + - When a Workload doesn't fit within the nominal quota of the ClusterQueue + and there are admitted Workloads in the ClusterQueue with lower priority. + Configured using withinClusterQueue. + - When a Workload may fit while both borrowing and preempting + low priority workloads in the Cohort. Configured using borrowWithinCohort. + - When FairSharing is enabled, to maintain fair distribution of + unused resources. See FairSharing documentation. + + The preemption algorithm tries to find a minimal set of Workloads to + preempt to accomomdate the pending Workload, preempting Workloads with + lower priority first. + properties: + borrowWithinCohort: + default: {} + description: |- + BorrowWithinCohort contains configuration which allows to preempt workloads + within cohort while borrowing. It only works with Classical Preemption, + __not__ with Fair Sharing. + properties: + maxPriorityThreshold: + description: |- + maxPriorityThreshold allows to restrict the set of workloads which + might be preempted by a borrowing workload, to only workloads with + priority less than or equal to the specified threshold priority. + When the threshold is not specified, then any workload satisfying the + policy can be preempted by the borrowing workload. + format: int32 + type: integer + policy: + default: Never + description: |- + policy determines the policy for preemption to reclaim quota within cohort while borrowing. + Possible values are: + - `Never` (default): do not allow for preemption, in other + ClusterQueues within the cohort, for a borrowing workload. + - `LowerPriority`: allow preemption, in other ClusterQueues + within the cohort, for a borrowing workload, but only if + the preempted workloads are of lower priority. + enum: + - Never + - LowerPriority + type: string + type: object + reclaimWithinCohort: + default: Never + description: |- + reclaimWithinCohort determines whether a pending Workload can preempt + Workloads from other ClusterQueues in the cohort that are using more than + their nominal quota. The possible values are: + + - `Never` (default): do not preempt Workloads in the cohort. + - `LowerPriority`: **Classic Preemption** if the pending Workload + fits within the nominal quota of its ClusterQueue, only preempt + Workloads in the cohort that have lower priority than the pending + Workload. **Fair Sharing** only preempt Workloads in the cohort that + have lower priority than the pending Workload and that satisfy the + Fair Sharing preemptionStategies. + - `Any`: **Classic Preemption** if the pending Workload fits within + the nominal quota of its ClusterQueue, preempt any Workload in the + cohort, irrespective of priority. **Fair Sharing** preempt Workloads + in the cohort that satisfy the Fair Sharing preemptionStrategies. + enum: + - Never + - LowerPriority + - Any + type: string + withinClusterQueue: + default: Never + description: |- + withinClusterQueue determines whether a pending Workload that doesn't fit + within the nominal quota for its ClusterQueue, can preempt active Workloads in + the ClusterQueue. The possible values are: + + - `Never` (default): do not preempt Workloads in the ClusterQueue. + - `LowerPriority`: only preempt Workloads in the ClusterQueue that have + lower priority than the pending Workload. + - `LowerOrNewerEqualPriority`: only preempt Workloads in the ClusterQueue that + either have a lower priority than the pending workload or equal priority + and are newer than the pending workload. + enum: + - Never + - LowerPriority + - LowerOrNewerEqualPriority + type: string + type: object + x-kubernetes-validations: + - message: reclaimWithinCohort=Never and borrowWithinCohort.Policy!=Never + rule: '!(self.reclaimWithinCohort == ''Never'' && has(self.borrowWithinCohort) + && self.borrowWithinCohort.policy != ''Never'')' + queueingStrategy: + default: BestEffortFIFO + description: |- + QueueingStrategy indicates the queueing strategy of the workloads + across the queues in this ClusterQueue. + Current Supported Strategies: + + - StrictFIFO: workloads are ordered strictly by creation time. + Older workloads that can't be admitted will block admitting newer + workloads even if they fit available quota. + - BestEffortFIFO: workloads are ordered by creation time, + however older workloads that can't be admitted will not block + admitting newer workloads that fit existing quota. + enum: + - StrictFIFO + - BestEffortFIFO + type: string + resourceGroups: + description: |- + resourceGroups describes groups of resources. + Each resource group defines the list of resources and a list of flavors + that provide quotas for these resources. + Each resource and each flavor can only form part of one resource group. + resourceGroups can be up to 16. + items: + properties: + coveredResources: + description: |- + coveredResources is the list of resources covered by the flavors in this + group. + Examples: cpu, memory, vendor.com/gpu. + The list cannot be empty and it can contain up to 16 resources. + items: + description: ResourceName is the name identifying various + resources in a ResourceList. + type: string + maxItems: 16 + minItems: 1 + type: array + flavors: + description: |- + flavors is the list of flavors that provide the resources of this group. + Typically, different flavors represent different hardware models + (e.g., gpu models, cpu architectures) or pricing models (on-demand vs spot + cpus). + Each flavor MUST list all the resources listed for this group in the same + order as the .resources field. + The list cannot be empty and it can contain up to 16 flavors. + items: + properties: + name: + description: |- + name of this flavor. The name should match the .metadata.name of a + ResourceFlavor. If a matching ResourceFlavor does not exist, the + ClusterQueue will have an Active condition set to False. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: |- + resources is the list of quotas for this flavor per resource. + There could be up to 16 resources. + items: + properties: + borrowingLimit: + anyOf: + - type: integer + - type: string + description: |- + borrowingLimit is the maximum amount of quota for the [flavor, resource] + combination that this ClusterQueue is allowed to borrow from the unused + quota of other ClusterQueues in the same cohort. + In total, at a given time, Workloads in a ClusterQueue can consume a + quantity of quota equal to nominalQuota+borrowingLimit, assuming the other + ClusterQueues in the cohort have enough unused quota. + If null, it means that there is no borrowing limit. + If not null, it must be non-negative. + borrowingLimit must be null if spec.cohort is empty. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + lendingLimit: + anyOf: + - type: integer + - type: string + description: |- + lendingLimit is the maximum amount of unused quota for the [flavor, resource] + combination that this ClusterQueue can lend to other ClusterQueues in the same cohort. + In total, at a given time, ClusterQueue reserves for its exclusive use + a quantity of quota equals to nominalQuota - lendingLimit. + If null, it means that there is no lending limit, meaning that + all the nominalQuota can be borrowed by other clusterQueues in the cohort. + If not null, it must be non-negative. + lendingLimit must be null if spec.cohort is empty. + This field is in beta stage and is enabled by default. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name of this resource. + type: string + nominalQuota: + anyOf: + - type: integer + - type: string + description: |- + nominalQuota is the quantity of this resource that is available for + Workloads admitted by this ClusterQueue at a point in time. + The nominalQuota must be non-negative. + nominalQuota should represent the resources in the cluster available for + running jobs (after discounting resources consumed by system components + and pods not managed by kueue). In an autoscaled cluster, nominalQuota + should account for resources that can be provided by a component such as + Kubernetes cluster-autoscaler. + + If the ClusterQueue belongs to a cohort, the sum of the quotas for each + (flavor, resource) combination defines the maximum quantity that can be + allocated by a ClusterQueue in the cohort. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + - nominalQuota + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - coveredResources + - flavors + type: object + x-kubernetes-validations: + - message: flavors must have the same number of resources as the + coveredResources + rule: self.flavors.all(x, size(x.resources) == size(self.coveredResources)) + maxItems: 16 + type: array + x-kubernetes-list-type: atomic + stopPolicy: + default: None + description: |- + stopPolicy - if set to a value different from None, the ClusterQueue is considered Inactive, no new reservation being + made. + + Depending on its value, its associated workloads will: + + - None - Workloads are admitted + - HoldAndDrain - Admitted workloads are evicted and Reserving workloads will cancel the reservation. + - Hold - Admitted workloads will run to completion and Reserving workloads will cancel the reservation. + enum: + - None + - Hold + - HoldAndDrain + type: string + type: object + x-kubernetes-validations: + - message: borrowingLimit must be nil when cohort is empty + rule: '!has(self.cohort) && has(self.resourceGroups) ? self.resourceGroups.all(rg, + rg.flavors.all(f, f.resources.all(r, !has(r.borrowingLimit)))) : true' + status: + description: ClusterQueueStatus defines the observed state of ClusterQueue + properties: + admittedWorkloads: + description: |- + admittedWorkloads is the number of workloads currently admitted to this + clusterQueue and haven't finished yet. + format: int32 + type: integer + conditions: + description: |- + conditions hold the latest available observations of the ClusterQueue + current state. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + fairSharing: + description: |- + fairSharing contains the current state for this ClusterQueue + when participating in Fair Sharing. + This is recorded only when Fair Sharing is enabled in the Kueue configuration. + properties: + admissionFairSharingStatus: + description: admissionFairSharingStatus represents information + relevant to the Admission Fair Sharing + properties: + consumedResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + ConsumedResources represents the aggregated usage of resources over time, + with decaying function applied. + The value is populated if usage consumption functionality is enabled in Kueue config. + type: object + lastUpdate: + description: LastUpdate is the time when share and consumed + resources were updated. + format: date-time + type: string + required: + - consumedResources + - lastUpdate + type: object + weightedShare: + description: |- + WeightedShare represents the maximum of the ratios of usage + above nominal quota to the lendable resources in the + Cohort, among all the resources provided by the Node, and + divided by the weight. If zero, it means that the usage of + the Node is below the nominal quota. If the Node has a + weight of zero and is borrowing, this will return + 9223372036854775807, the maximum possible share value. + format: int64 + type: integer + required: + - weightedShare + type: object + flavorsReservation: + description: |- + flavorsReservation are the reserved quotas, by flavor, currently in use by the + workloads assigned to this ClusterQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: resources lists the quota usage for the resources + in this flavor. + items: + properties: + borrowed: + anyOf: + - type: integer + - type: string + description: |- + Borrowed is quantity of quota that is borrowed from the cohort. In other + words, it's the used quota that is over the nominalQuota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name of the resource + type: string + total: + anyOf: + - type: integer + - type: string + description: |- + total is the total quantity of used quota, including the amount borrowed + from the cohort. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + flavorsUsage: + description: |- + flavorsUsage are the used quotas, by flavor, currently in use by the + workloads admitted in this ClusterQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: resources lists the quota usage for the resources + in this flavor. + items: + properties: + borrowed: + anyOf: + - type: integer + - type: string + description: |- + Borrowed is quantity of quota that is borrowed from the cohort. In other + words, it's the used quota that is over the nominalQuota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name of the resource + type: string + total: + anyOf: + - type: integer + - type: string + description: |- + total is the total quantity of used quota, including the amount borrowed + from the cohort. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + pendingWorkloads: + description: |- + pendingWorkloads is the number of workloads currently waiting to be + admitted to this clusterQueue. + format: int32 + type: integer + pendingWorkloadsStatus: + description: |- + PendingWorkloadsStatus contains the information exposed about the current + status of the pending workloads in the cluster queue. + Deprecated: This field will be removed on v1beta2, use VisibilityOnDemand + (https://kueue.sigs.k8s.io/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand/) + instead. + properties: + clusterQueuePendingWorkload: + description: Head contains the list of top pending workloads. + items: + description: |- + ClusterQueuePendingWorkload contains the information identifying a pending workload + in the cluster queue. + properties: + name: + description: Name indicates the name of the pending workload. + type: string + namespace: + description: Namespace indicates the name of the pending + workload. + type: string + required: + - name + - namespace + type: object + type: array + x-kubernetes-list-type: atomic + lastChangeTime: + description: LastChangeTime indicates the time of the last change + of the structure. + format: date-time + type: string + required: + - lastChangeTime + type: object + reservingWorkloads: + description: |- + reservingWorkloads is the number of workloads currently reserving quota in this + clusterQueue. + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: cohorts.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: Cohort + listKind: CohortList + plural: cohorts + singular: cohort + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + Cohort defines the Cohorts API. + + Hierarchical Cohorts (any Cohort which has a parent) are compatible + with Fair Sharing as of v0.11. Using these features together in + V0.9 and V0.10 is unsupported, and results in undefined behavior. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CohortSpec defines the desired state of Cohort + properties: + fairSharing: + description: |- + fairSharing defines the properties of the Cohort when + participating in FairSharing. The values are only relevant + if FairSharing is enabled in the Kueue configuration. + properties: + weight: + anyOf: + - type: integer + - type: string + default: 1 + description: |- + weight gives a comparative advantage to this ClusterQueue + or Cohort when competing for unused resources in the + Cohort. The share is based on the dominant resource usage + above nominal quotas for each resource, divided by the + weight. Admission prioritizes scheduling workloads from + ClusterQueues and Cohorts with the lowest share and + preempting workloads from the ClusterQueues and Cohorts + with the highest share. A zero weight implies infinite + share value, meaning that this Node will always be at + disadvantage against other ClusterQueues and Cohorts. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + parent: + description: |- + Parent references the name of the Cohort's parent, if + any. It satisfies one of three cases: + 1) Unset. This Cohort is the root of its Cohort tree. + 2) References a non-existent Cohort. We use default Cohort (no borrowing/lending limits). + 3) References an existent Cohort. + + If a cycle is created, we disable all members of the + Cohort, including ClusterQueues, until the cycle is + removed. We prevent further admission while the cycle + exists. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resourceGroups: + description: |- + ResourceGroups describes groupings of Resources and + Flavors. Each ResourceGroup defines a list of Resources + and a list of Flavors which provide quotas for these + Resources. Each Resource and each Flavor may only form part + of one ResourceGroup. There may be up to 16 ResourceGroups + within a Cohort. + + BorrowingLimit limits how much members of this Cohort + subtree can borrow from the parent subtree. + + LendingLimit limits how much members of this Cohort subtree + can lend to the parent subtree. + + Borrowing and Lending limits must only be set when the + Cohort has a parent. Otherwise, the Cohort create/update + will be rejected by the webhook. + items: + properties: + coveredResources: + description: |- + coveredResources is the list of resources covered by the flavors in this + group. + Examples: cpu, memory, vendor.com/gpu. + The list cannot be empty and it can contain up to 16 resources. + items: + description: ResourceName is the name identifying various + resources in a ResourceList. + type: string + maxItems: 16 + minItems: 1 + type: array + flavors: + description: |- + flavors is the list of flavors that provide the resources of this group. + Typically, different flavors represent different hardware models + (e.g., gpu models, cpu architectures) or pricing models (on-demand vs spot + cpus). + Each flavor MUST list all the resources listed for this group in the same + order as the .resources field. + The list cannot be empty and it can contain up to 16 flavors. + items: + properties: + name: + description: |- + name of this flavor. The name should match the .metadata.name of a + ResourceFlavor. If a matching ResourceFlavor does not exist, the + ClusterQueue will have an Active condition set to False. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: |- + resources is the list of quotas for this flavor per resource. + There could be up to 16 resources. + items: + properties: + borrowingLimit: + anyOf: + - type: integer + - type: string + description: |- + borrowingLimit is the maximum amount of quota for the [flavor, resource] + combination that this ClusterQueue is allowed to borrow from the unused + quota of other ClusterQueues in the same cohort. + In total, at a given time, Workloads in a ClusterQueue can consume a + quantity of quota equal to nominalQuota+borrowingLimit, assuming the other + ClusterQueues in the cohort have enough unused quota. + If null, it means that there is no borrowing limit. + If not null, it must be non-negative. + borrowingLimit must be null if spec.cohort is empty. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + lendingLimit: + anyOf: + - type: integer + - type: string + description: |- + lendingLimit is the maximum amount of unused quota for the [flavor, resource] + combination that this ClusterQueue can lend to other ClusterQueues in the same cohort. + In total, at a given time, ClusterQueue reserves for its exclusive use + a quantity of quota equals to nominalQuota - lendingLimit. + If null, it means that there is no lending limit, meaning that + all the nominalQuota can be borrowed by other clusterQueues in the cohort. + If not null, it must be non-negative. + lendingLimit must be null if spec.cohort is empty. + This field is in beta stage and is enabled by default. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name of this resource. + type: string + nominalQuota: + anyOf: + - type: integer + - type: string + description: |- + nominalQuota is the quantity of this resource that is available for + Workloads admitted by this ClusterQueue at a point in time. + The nominalQuota must be non-negative. + nominalQuota should represent the resources in the cluster available for + running jobs (after discounting resources consumed by system components + and pods not managed by kueue). In an autoscaled cluster, nominalQuota + should account for resources that can be provided by a component such as + Kubernetes cluster-autoscaler. + + If the ClusterQueue belongs to a cohort, the sum of the quotas for each + (flavor, resource) combination defines the maximum quantity that can be + allocated by a ClusterQueue in the cohort. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + - nominalQuota + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - coveredResources + - flavors + type: object + x-kubernetes-validations: + - message: flavors must have the same number of resources as the + coveredResources + rule: self.flavors.all(x, size(x.resources) == size(self.coveredResources)) + maxItems: 16 + type: array + x-kubernetes-list-type: atomic + type: object + status: + description: CohortStatus defines the observed state of Cohort. + properties: + fairSharing: + description: |- + fairSharing contains the current state for this Cohort + when participating in Fair Sharing. + The is recorded only when Fair Sharing is enabled in the Kueue configuration. + properties: + admissionFairSharingStatus: + description: admissionFairSharingStatus represents information + relevant to the Admission Fair Sharing + properties: + consumedResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + ConsumedResources represents the aggregated usage of resources over time, + with decaying function applied. + The value is populated if usage consumption functionality is enabled in Kueue config. + type: object + lastUpdate: + description: LastUpdate is the time when share and consumed + resources were updated. + format: date-time + type: string + required: + - consumedResources + - lastUpdate + type: object + weightedShare: + description: |- + WeightedShare represents the maximum of the ratios of usage + above nominal quota to the lendable resources in the + Cohort, among all the resources provided by the Node, and + divided by the weight. If zero, it means that the usage of + the Node is below the nominal quota. If the Node has a + weight of zero and is borrowing, this will return + 9223372036854775807, the maximum possible share value. + format: int64 + type: integer + required: + - weightedShare + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: localqueues.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: LocalQueue + listKind: LocalQueueList + plural: localqueues + shortNames: + - queue + - queues + - lq + singular: localqueue + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Backing ClusterQueue + jsonPath: .spec.clusterQueue + name: ClusterQueue + type: string + - description: Number of pending workloads + jsonPath: .status.pendingWorkloads + name: Pending Workloads + type: integer + - description: Number of admitted workloads that haven't finished yet. + jsonPath: .status.admittedWorkloads + name: Admitted Workloads + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + description: LocalQueue is the Schema for the localQueues API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: LocalQueueSpec defines the desired state of LocalQueue + properties: + clusterQueue: + description: clusterQueue is a reference to a clusterQueue that backs + this localQueue. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + fairSharing: + description: |- + fairSharing defines the properties of the LocalQueue when + participating in AdmissionFairSharing. The values are only relevant + if AdmissionFairSharing is enabled in the Kueue configuration. + properties: + weight: + anyOf: + - type: integer + - type: string + default: 1 + description: |- + weight gives a comparative advantage to this ClusterQueue + or Cohort when competing for unused resources in the + Cohort. The share is based on the dominant resource usage + above nominal quotas for each resource, divided by the + weight. Admission prioritizes scheduling workloads from + ClusterQueues and Cohorts with the lowest share and + preempting workloads from the ClusterQueues and Cohorts + with the highest share. A zero weight implies infinite + share value, meaning that this Node will always be at + disadvantage against other ClusterQueues and Cohorts. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + stopPolicy: + default: None + description: |- + stopPolicy - if set to a value different from None, the LocalQueue is considered Inactive, + no new reservation being made. + + Depending on its value, its associated workloads will: + + - None - Workloads are admitted + - HoldAndDrain - Admitted workloads are evicted and Reserving workloads will cancel the reservation. + - Hold - Admitted workloads will run to completion and Reserving workloads will cancel the reservation. + enum: + - None + - Hold + - HoldAndDrain + type: string + type: object + status: + description: LocalQueueStatus defines the observed state of LocalQueue + properties: + admittedWorkloads: + description: |- + admittedWorkloads is the number of workloads in this LocalQueue + admitted to a ClusterQueue and that haven't finished yet. + format: int32 + type: integer + conditions: + description: |- + Conditions hold the latest available observations of the LocalQueue + current state. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + fairSharing: + description: FairSharing contains the information about the current + status of fair sharing. + properties: + admissionFairSharingStatus: + description: admissionFairSharingStatus represents information + relevant to the Admission Fair Sharing + properties: + consumedResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + ConsumedResources represents the aggregated usage of resources over time, + with decaying function applied. + The value is populated if usage consumption functionality is enabled in Kueue config. + type: object + lastUpdate: + description: LastUpdate is the time when share and consumed + resources were updated. + format: date-time + type: string + required: + - consumedResources + - lastUpdate + type: object + weightedShare: + description: |- + WeightedShare represents the maximum of the ratios of usage + above nominal quota to the lendable resources in the + Cohort, among all the resources provided by the Node, and + divided by the weight. If zero, it means that the usage of + the Node is below the nominal quota. If the Node has a + weight of zero and is borrowing, this will return + 9223372036854775807, the maximum possible share value. + format: int64 + type: integer + required: + - weightedShare + type: object + flavorUsage: + description: |- + flavorsUsage are the used quotas, by flavor currently in use by the + workloads assigned to this LocalQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: resources lists the quota usage for the resources + in this flavor. + items: + properties: + name: + description: name of the resource. + type: string + total: + anyOf: + - type: integer + - type: string + description: total is the total quantity of used quota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + flavors: + description: flavors lists all currently available ResourceFlavors + in specified ClusterQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + nodeLabels: + additionalProperties: + type: string + description: |- + nodeLabels are labels that associate the ResourceFlavor with Nodes that + have the same labels. + maxProperties: 8 + type: object + x-kubernetes-map-type: atomic + nodeTaints: + description: |- + nodeTaints are taints that the nodes associated with this ResourceFlavor + have. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to + a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint + key. + type: string + required: + - effect + - key + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + resources: + description: resources used in the flavor. + items: + description: ResourceName is the name identifying various + resources in a ResourceList. + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + topology: + description: |- + topology is the topology that associated with this ResourceFlavor. + + This is an alpha field and requires enabling the TopologyAwareScheduling + feature gate. + properties: + levels: + description: levels define the levels of topology. + items: + type: string + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + name: + description: name is the name of the topology. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - levels + - name + type: object + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + flavorsReservation: + description: |- + flavorsReservation are the reserved quotas, by flavor currently in use by the + workloads assigned to this LocalQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: resources lists the quota usage for the resources + in this flavor. + items: + properties: + name: + description: name of the resource. + type: string + total: + anyOf: + - type: integer + - type: string + description: total is the total quantity of used quota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + pendingWorkloads: + description: PendingWorkloads is the number of Workloads in the LocalQueue + not yet admitted to a ClusterQueue + format: int32 + type: integer + reservingWorkloads: + description: |- + reservingWorkloads is the number of workloads in this LocalQueue + reserving quota in a ClusterQueue and that haven't finished yet. + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: multikueueclusters.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: MultiKueueCluster + listKind: MultiKueueClusterList + plural: multikueueclusters + singular: multikueuecluster + scope: Cluster + versions: + - additionalPrinterColumns: + - description: MultiKueueCluster is connected + jsonPath: .status.conditions[?(@.type=='Active')].status + name: Connected + type: string + - description: Time this workload was created + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: MultiKueueCluster is the Schema for the multikueue API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + kubeConfig: + description: Information how to connect to the cluster. + properties: + location: + description: |- + Location of the KubeConfig. + + If LocationType is Secret then Location is the name of the secret inside the namespace in + which the kueue controller manager is running. The config should be stored in the "kubeconfig" key. + type: string + locationType: + default: Secret + description: Type of the KubeConfig location. + enum: + - Secret + - Path + type: string + required: + - location + - locationType + type: object + required: + - kubeConfig + type: object + status: + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: multikueueconfigs.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: MultiKueueConfig + listKind: MultiKueueConfigList + plural: multikueueconfigs + singular: multikueueconfig + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: MultiKueueConfig is the Schema for the multikueue API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: MultiKueueConfigSpec defines the desired state of MultiKueueConfig + properties: + clusters: + description: List of MultiKueueClusters names where the workloads + from the ClusterQueue should be distributed. + items: + type: string + maxItems: 10 + minItems: 1 + type: array + x-kubernetes-list-type: set + required: + - clusters + type: object + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: provisioningrequestconfigs.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: ProvisioningRequestConfig + listKind: ProvisioningRequestConfigList + plural: provisioningrequestconfigs + singular: provisioningrequestconfig + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: ProvisioningRequestConfig is the Schema for the provisioningrequestconfig + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ProvisioningRequestConfigSpec defines the desired state of + ProvisioningRequestConfig + properties: + managedResources: + description: |- + managedResources contains the list of resources managed by the autoscaling. + + If empty, all resources are considered managed. + + If not empty, the ProvisioningRequest will contain only the podsets that are + requesting at least one of them. + + If none of the workloads podsets is requesting at least a managed resource, + the workload is considered ready. + items: + description: ResourceName is the name identifying various resources + in a ResourceList. + type: string + maxItems: 100 + type: array + x-kubernetes-list-type: set + parameters: + additionalProperties: + description: Parameter is limited to 255 characters. + maxLength: 255 + type: string + description: Parameters contains all other parameters classes may + require. + maxProperties: 100 + type: object + podSetMergePolicy: + description: |- + podSetMergePolicy specifies the policy for merging PodSets before being passed + to the cluster autoscaler. + enum: + - IdenticalPodTemplates + - IdenticalWorkloadSchedulingRequirements + type: string + podSetUpdates: + description: |- + podSetUpdates specifies the update of the workload's PodSetUpdates which + are used to target the provisioned nodes. + properties: + nodeSelector: + description: nodeSelector specifies the list of updates for the + NodeSelector. + items: + properties: + key: + description: key specifies the key for the NodeSelector. + maxLength: 317 + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + valueFromProvisioningClassDetail: + description: |- + valueFromProvisioningClassDetail specifies the key of the + ProvisioningRequest.status.provisioningClassDetails from which the value + is used for the update. + maxLength: 32768 + minLength: 1 + type: string + required: + - key + - valueFromProvisioningClassDetail + type: object + maxItems: 8 + type: array + type: object + provisioningClassName: + description: |- + ProvisioningClassName describes the different modes of provisioning the resources. + Check autoscaling.x-k8s.io ProvisioningRequestSpec.ProvisioningClassName for details. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + retryStrategy: + default: + backoffBaseSeconds: 60 + backoffLimitCount: 3 + backoffMaxSeconds: 1800 + description: |- + retryStrategy defines strategy for retrying ProvisioningRequest. + If null, then the default configuration is applied with the following parameter values: + backoffLimitCount: 3 + backoffBaseSeconds: 60 - 1 min + backoffMaxSeconds: 1800 - 30 mins + + To switch off retry mechanism + set retryStrategy.backoffLimitCount to 0. + properties: + backoffBaseSeconds: + default: 60 + description: |- + BackoffBaseSeconds defines the base for the exponential backoff for + re-queuing an evicted workload. + + Defaults to 60. + format: int32 + type: integer + backoffLimitCount: + default: 3 + description: |- + BackoffLimitCount defines the maximum number of re-queuing retries. + Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). + + Every backoff duration is about "b*2^(n-1)+Rand" where: + - "b" represents the base set by "BackoffBaseSeconds" parameter, + - "n" represents the "workloadStatus.requeueState.count", + - "Rand" represents the random jitter. + During this time, the workload is taken as an inadmissible and + other workloads will have a chance to be admitted. + By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...). + + Defaults to 3. + format: int32 + type: integer + backoffMaxSeconds: + default: 1800 + description: |- + BackoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload. + + Defaults to 1800. + format: int32 + type: integer + type: object + required: + - provisioningClassName + type: object + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: resourceflavors.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: ResourceFlavor + listKind: ResourceFlavorList + plural: resourceflavors + shortNames: + - flavor + - flavors + - rf + singular: resourceflavor + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: ResourceFlavor is the Schema for the resourceflavors API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ResourceFlavorSpec defines the desired state of the ResourceFlavor + properties: + nodeLabels: + additionalProperties: + type: string + description: |- + nodeLabels are labels that associate the ResourceFlavor with Nodes that + have the same labels. + When a Workload is admitted, its podsets can only get assigned + ResourceFlavors whose nodeLabels match the nodeSelector and nodeAffinity + fields. + Once a ResourceFlavor is assigned to a podSet, the ResourceFlavor's + nodeLabels should be injected into the pods of the Workload by the + controller that integrates with the Workload object. + + nodeLabels can be up to 8 elements. + maxProperties: 8 + type: object + x-kubernetes-map-type: atomic + nodeTaints: + description: |- + nodeTaints are taints that the nodes associated with this ResourceFlavor + have. + Workloads' podsets must have tolerations for these nodeTaints in order to + get assigned this ResourceFlavor during admission. + When this ResourceFlavor has also set the matching tolerations (in .spec.tolerations), + then the nodeTaints are not considered during admission. + Only the 'NoSchedule' and 'NoExecute' taint effects are evaluated, + while 'PreferNoSchedule' is ignored. + + An example of a nodeTaint is + cloud.provider.com/preemptible="true":NoSchedule + + nodeTaints can be up to 8 elements. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: 'supported taint effect values: ''NoSchedule'', ''PreferNoSchedule'', + ''NoExecute''' + rule: self.all(x, x.effect in ['NoSchedule', 'PreferNoSchedule', + 'NoExecute']) + tolerations: + description: |- + tolerations are extra tolerations that will be added to the pods admitted in + the quota associated with this resource flavor. + + An example of a toleration is + cloud.provider.com/preemptible="true":NoSchedule + + tolerations can be up to 8 elements. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: operator must be Exists when 'key' is empty, which means + 'match all values and all keys' + rule: 'self.all(x, !has(x.key) ? x.operator == ''Exists'' : true)' + - message: effect must be 'NoExecute' when 'tolerationSeconds' is + set + rule: 'self.all(x, has(x.tolerationSeconds) ? x.effect == ''NoExecute'' + : true)' + - message: 'supported toleration values: ''Equal''(default), ''Exists''' + rule: self.all(x, !has(x.operator) || x.operator in ['Equal', 'Exists']) + - message: a value must be empty when 'operator' is 'Exists' + rule: 'self.all(x, has(x.operator) && x.operator == ''Exists'' ? + !has(x.value) : true)' + - message: 'supported taint effect values: ''NoSchedule'', ''PreferNoSchedule'', + ''NoExecute''' + rule: self.all(x, !has(x.effect) || x.effect in ['NoSchedule', 'PreferNoSchedule', + 'NoExecute']) + topologyName: + description: |- + topologyName indicates topology for the TAS ResourceFlavor. + When specified, it enables scraping of the topology information from the + nodes matching to the Resource Flavor node labels. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + type: object + x-kubernetes-validations: + - message: at least one nodeLabel is required when topology is set + rule: '!has(self.topologyName) || self.nodeLabels.size() >= 1' + - message: resourceFlavorSpec are immutable when topologyName is set + rule: '!has(oldSelf.topologyName) || self == oldSelf' + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: topologies.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: Topology + listKind: TopologyList + plural: topologies + singular: topology + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Topology is the Schema for the topology API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: TopologySpec defines the desired state of Topology + properties: + levels: + description: levels define the levels of topology. + items: + description: TopologyLevel defines the desired state of TopologyLevel + properties: + nodeLabel: + description: |- + nodeLabel indicates the name of the node label for a specific topology + level. + + Examples: + - cloud.provider.com/topology-block + - cloud.provider.com/topology-rack + maxLength: 316 + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - nodeLabel + type: object + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + - message: must be unique + rule: size(self.filter(i, size(self.filter(j, j == i)) > 1)) == + 0 + - message: the kubernetes.io/hostname label can only be used at the + lowest level of topology + rule: size(self.filter(i, i.nodeLabel == 'kubernetes.io/hostname')) + == 0 || self[size(self) - 1].nodeLabel == 'kubernetes.io/hostname' + required: + - levels + type: object + required: + - spec + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: workloadpriorityclasses.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: WorkloadPriorityClass + listKind: WorkloadPriorityClassList + plural: workloadpriorityclasses + singular: workloadpriorityclass + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Value of workloadPriorityClass's Priority + jsonPath: .value + name: Value + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + description: WorkloadPriorityClass is the Schema for the workloadPriorityClass + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + description: + description: |- + description is an arbitrary string that usually provides guidelines on + when this workloadPriorityClass should be used. + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + value: + description: |- + value represents the integer value of this workloadPriorityClass. This is the actual priority that workloads + receive when jobs have the name of this class in their workloadPriorityClass label. + Changing the value of workloadPriorityClass doesn't affect the priority of workloads that were already created. + format: int32 + type: integer + required: + - value + type: object + served: true + storage: true + subresources: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: workloads.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: Workload + listKind: WorkloadList + plural: workloads + shortNames: + - wl + singular: workload + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Name of the queue this workload was submitted to + jsonPath: .spec.queueName + name: Queue + type: string + - description: Name of the ClusterQueue where the workload is reserving quota + jsonPath: .status.admission.clusterQueue + name: Reserved in + type: string + - description: Admission status + jsonPath: .status.conditions[?(@.type=='Admitted')].status + name: Admitted + type: string + - description: Workload finished + jsonPath: .status.conditions[?(@.type=='Finished')].status + name: Finished + type: string + - description: Time this workload was created + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Workload is the Schema for the workloads API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: WorkloadSpec defines the desired state of Workload + properties: + active: + default: true + description: |- + Active determines if a workload can be admitted into a queue. + Changing active from true to false will evict any running workloads. + Possible values are: + + - false: indicates that a workload should never be admitted and evicts running workloads + - true: indicates that a workload can be evaluated for admission into it's respective queue. + + Defaults to true + type: boolean + maximumExecutionTimeSeconds: + description: |- + maximumExecutionTimeSeconds if provided, determines the maximum time, in seconds, + the workload can be admitted before it's automatically deactivated. + + If unspecified, no execution time limit is enforced on the Workload. + format: int32 + minimum: 1 + type: integer + podSets: + description: |- + podSets is a list of sets of homogeneous pods, each described by a Pod spec + and a count. + There must be at least one element and at most 8. + podSets cannot be changed. + items: + properties: + count: + default: 1 + description: count is the number of pods for the spec. + format: int32 + minimum: 0 + type: integer + minCount: + description: |- + minCount is the minimum number of pods for the spec acceptable + if the workload supports partial admission. + + If not provided, partial admission for the current PodSet is not + enabled. + + Only one podSet within the workload can use this. + + This is an alpha field and requires enabling PartialAdmission feature gate. + format: int32 + minimum: 1 + type: integer + name: + default: main + description: name is the PodSet name. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + template: + description: |- + template is the Pod template. + + The only allowed fields in template.metadata are labels and annotations. + + If requests are omitted for a container or initContainer, + they default to the limits if they are explicitly specified for the + container or initContainer. + + During admission, the rules in nodeSelector and + nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution that match + the keys in the nodeLabels from the ResourceFlavors considered for this + Workload are used to filter the ResourceFlavors that can be assigned to + this podSet. + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling + rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, + in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules + (e.g. co-locate this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the same + node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates + whether a service account token should be automatically + mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container that you + want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if value + is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command + to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP + GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration + that the container should sleep. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command + to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP + GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration + that the container should sleep. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name of + a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of + a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS resolver + options of a pod. + properties: + name: + description: |- + Name is this DNS resolver option's name. + Required. + type: string + value: + description: Value is this DNS resolver option's + value. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if value + is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed for ephemeral + containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command + to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP + GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration + that the container should sleep. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command + to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP + GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration + that the container should sleep. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral + containers. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + This may only be set for init containers. You cannot set this field on + ephemeral containers. + type: string + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name of + a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of + a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + If this option is set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + of that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container that you + want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if value + is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command + to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP + GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration + that the container should sleep. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command + to execute in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP + GET request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration + that the container should sleep. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies a GRPC HealthCheckRequest. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies an HTTP GET + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a + custom header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies a connection + to a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name of + a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of + a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the reference + to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition + in the pod's condition list with matching type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + resources: + description: |- + Resources is the total amount of CPU and Memory resources required by all + containers in the pod. It supports specifying Requests and Limits for + "cpu" and "memory" resource names only. ResourceClaims are not supported. + + This field enables fine-grained control over resource allocation for the + entire pod, allowing resource sharing among containers in a pod. + + This is an alpha field and requires enabling the PodLevelResources feature + gate. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated to a + Pod to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that + applies to the container. + type: string + role: + description: Role is a SELinux role label that + applies to the container. + type: string + type: + description: Type is a SELinux type label that + applies to the container. + type: string + user: + description: User is a SELinux user label that + applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter + to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies how + to spread matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume in a + pod that may be accessed by any container in the + pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + Deprecated: AWSElasticBlockStore is deprecated. All operations for the in-tree + awsElasticBlockStore type are redirected to the ebs.csi.aws.com CSI driver. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: |- + azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. + Deprecated: AzureDisk is deprecated. All operations for the in-tree azureDisk type + are redirected to the disk.csi.azure.com CSI driver. + properties: + cachingMode: + description: 'cachingMode is the Host Caching + mode: None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data + disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk + in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: + multiple blob disks per storage account Dedicated: + single blob disk per storage account Managed: + azure managed data disk (only in managed + availability set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: |- + azureFile represents an Azure File Service mount on the host and bind mount to the pod. + Deprecated: AzureFile is deprecated. All operations for the in-tree azureFile type + are redirected to the file.csi.azure.com CSI driver. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret + that contains Azure Storage Account Name + and Key + type: string + shareName: + description: shareName is the azure share + Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: |- + cephFS represents a Ceph FS mount on the host that shares a pod's lifetime. + Deprecated: CephFS is deprecated and the in-tree cephfs type is no longer supported. + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as the + mounted root, rather than the full Ceph + tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + Deprecated: Cinder is deprecated. All operations for the in-tree cinder type + are redirected to the cinder.csi.openstack.org CSI driver. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap + that should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the + ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) + represents ephemeral storage that is handled + by certain external CSI drivers. + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API + about the pod that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API + volume file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file to + be created. Must not be absolute or + contain the ''..'' path. Must be utf-8 + encoded. The first item of the relative + path must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query + over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding + reference to the PersistentVolume + backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource + that is attached to a kubelet's host machine + and then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target lun + number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target + worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + Deprecated: FlexVolume is deprecated. Consider using a CSIDriver instead. + properties: + driver: + description: driver is the name of the driver + to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field + holds extra command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: |- + flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running. + Deprecated: Flocker is deprecated and the in-tree flocker type is no longer supported. + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the + dataset. This is unique identifier of a + Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + Deprecated: GCEPersistentDisk is deprecated. All operations for the in-tree + gcePersistentDisk type are redirected to the pd.csi.storage.gke.io CSI driver. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + Deprecated: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for + the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether + support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether + support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified + Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun + number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret + for iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: |- + photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine. + Deprecated: PhotonPersistentDisk is deprecated and the in-tree photonPersistentDisk type is no longer supported. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: |- + portworxVolume represents a portworx volume attached and mounted on kubelets host machine. + Deprecated: PortworxVolume is deprecated. All operations for the in-tree portworxVolume type + are redirected to the pxd.portworx.com CSI driver when the CSIMigrationPortworx feature-gate + is on. + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies + a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources + secrets, configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from + the volume root to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about + the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key + to a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether + the ConfigMap or its keys must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information + about the downwardAPI data to project + properties: + items: + description: Items is a list of + DownwardAPIVolume file + items: + description: DownwardAPIVolumeFile + represents information to create + the file containing the pod + field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only + annotations, labels, name, + namespace and uid are supported.' + properties: + apiVersion: + description: Version of + the schema the FieldPath + is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the + field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path + is the relative path name + of the file to be created. + Must not be absolute or + contain the ''..'' path. + Must be utf-8 encoded. The + first item of the relative + path must not start with + ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies + the output format of + the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information about + the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key + to a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify + whether the Secret or its key + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is + information about the serviceAccountToken + data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: |- + quobyte represents a Quobyte mount on the host that shares a pod's lifetime. + Deprecated: Quobyte is deprecated and the in-tree quobyte type is no longer supported. + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references + an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: |- + scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. + Deprecated: ScaleIO is deprecated and the in-tree scaleIO type is no longer supported. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of + the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name + of the ScaleIO Protection Domain for the + configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default + false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage + Pool associated with the protection domain. + type: string + system: + description: system is the name of the storage + system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether + the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: |- + storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. + Deprecated: StorageOS is deprecated and the in-tree storageos type is no longer supported. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: |- + vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine. + Deprecated: VsphereVolume is deprecated. All operations for the in-tree vsphereVolume type + are redirected to the csi.vsphere.vmware.com CSI driver. + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage + Policy Based Management (SPBM) profile ID + associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage + Policy Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies + vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object + topologyRequest: + description: topologyRequest defines the topology request for + the PodSet. + properties: + podIndexLabel: + description: |- + PodIndexLabel indicates the name of the label indexing the pods. + For example, in the context of + - kubernetes job this is: kubernetes.io/job-completion-index + - JobSet: kubernetes.io/job-completion-index (inherited from Job) + - Kubeflow: training.kubeflow.org/replica-index + type: string + preferred: + description: |- + preferred indicates the topology level preferred by the PodSet, as + indicated by the `kueue.x-k8s.io/podset-preferred-topology` PodSet + annotation. + type: string + required: + description: |- + required indicates the topology level required by the PodSet, as + indicated by the `kueue.x-k8s.io/podset-required-topology` PodSet + annotation. + type: string + subGroupCount: + description: |- + SubGroupIndexLabel indicates the count of replicated Jobs (groups) within a PodSet. + For example, in the context of JobSet this value is read from jobset.sigs.k8s.io/replicatedjob-replicas. + format: int32 + type: integer + subGroupIndexLabel: + description: |- + SubGroupIndexLabel indicates the name of the label indexing the instances of replicated Jobs (groups) + within a PodSet. For example, in the context of JobSet this is jobset.sigs.k8s.io/job-index. + type: string + unconstrained: + description: |- + unconstrained indicates that Kueue has the freedom to schedule the PodSet within + the entire available capacity, without constraints on the compactness of the placement. + This is indicated by the `kueue.x-k8s.io/podset-unconstrained-topology` PodSet annotation. + type: boolean + type: object + required: + - count + - template + type: object + x-kubernetes-validations: + - message: minCount should be positive and less or equal to count + rule: 'has(self.minCount) ? self.minCount <= self.count : true' + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + priority: + description: |- + Priority determines the order of access to the resources managed by the + ClusterQueue where the workload is queued. + The priority value is populated from PriorityClassName. + The higher the value, the higher the priority. + If priorityClassName is specified, priority must not be null. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the workload's priority. + "system-node-critical" and "system-cluster-critical" are two special + keywords which indicate the highest priorities with the former being + the highest priority. Any other name must be defined by creating a + PriorityClass object with that name. If not specified, the workload + priority will be default or zero if there is no default. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + priorityClassSource: + default: "" + description: |- + priorityClassSource determines whether the priorityClass field refers to a pod PriorityClass or kueue.x-k8s.io/workloadpriorityclass. + Workload's PriorityClass can accept the name of a pod priorityClass or a workloadPriorityClass. + When using pod PriorityClass, a priorityClassSource field has the scheduling.k8s.io/priorityclass value. + enum: + - kueue.x-k8s.io/workloadpriorityclass + - scheduling.k8s.io/priorityclass + - "" + type: string + queueName: + description: |- + queueName is the name of the LocalQueue the Workload is associated with. + queueName cannot be changed while .status.admission is not null. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - podSets + type: object + x-kubernetes-validations: + - message: priority should not be nil when priorityClassName is set + rule: 'has(self.priorityClassName) ? has(self.priority) : true' + status: + description: WorkloadStatus defines the observed state of Workload + properties: + accumulatedPastExexcutionTimeSeconds: + description: |- + accumulatedPastExexcutionTimeSeconds holds the total time, in seconds, the workload spent + in Admitted state, in the previous `Admit` - `Evict` cycles. + format: int32 + type: integer + admission: + description: |- + admission holds the parameters of the admission of the workload by a + ClusterQueue. admission can be set back to null, but its fields cannot be + changed once set. + properties: + clusterQueue: + description: clusterQueue is the name of the ClusterQueue that + admitted this workload. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + podSetAssignments: + description: PodSetAssignments hold the admission results for + each of the .spec.podSets entries. + items: + properties: + count: + description: |- + count is the number of pods taken into account at admission time. + This field will not change in case of quota reclaim. + Value could be missing for Workloads created before this field was added, + in that case spec.podSets[*].count value will be used. + format: int32 + minimum: 0 + type: integer + delayedTopologyRequest: + description: |- + delayedTopologyRequest indicates the topology assignment is delayed. + Topology assignment might be delayed in case there is ProvisioningRequest + AdmissionCheck used. + Kueue schedules the second pass of scheduling for each workload with at + least one PodSet which has delayedTopologyRequest=true and without + topologyAssignment. + type: string + flavors: + additionalProperties: + description: ResourceFlavorReference is the name of the + ResourceFlavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + description: Flavors are the flavors assigned to the workload + for each resource. + type: object + name: + default: main + description: Name is the name of the podSet. It should match + one of the names in .spec.podSets. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + resourceUsage: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + resourceUsage keeps track of the total resources all the pods in the podset need to run. + + Beside what is provided in podSet's specs, this calculation takes into account + the LimitRange defaults and RuntimeClass overheads at the moment of admission. + This field will not change in case of quota reclaim. + type: object + topologyAssignment: + description: |- + topologyAssignment indicates the topology assignment divided into + topology domains corresponding to the lowest level of the topology. + The assignment specifies the number of Pods to be scheduled per topology + domain and specifies the node selectors for each topology domain, in the + following way: the node selector keys are specified by the levels field + (same for all domains), and the corresponding node selector value is + specified by the domains.values subfield. If the TopologySpec.Levels field contains + "kubernetes.io/hostname" label, topologyAssignment will contain data only for + this label, and omit higher levels in the topology + + Example: + + topologyAssignment: + levels: + - cloud.provider.com/topology-block + - cloud.provider.com/topology-rack + domains: + - values: [block-1, rack-1] + count: 4 + - values: [block-1, rack-2] + count: 2 + + Here: + - 4 Pods are to be scheduled on nodes matching the node selector: + cloud.provider.com/topology-block: block-1 + cloud.provider.com/topology-rack: rack-1 + - 2 Pods are to be scheduled on nodes matching the node selector: + cloud.provider.com/topology-block: block-1 + cloud.provider.com/topology-rack: rack-2 + + Example: + Below there is an equivalent of the above example assuming, Topology + object defines kubernetes.io/hostname as the lowest level in topology. + Hence we omit higher level of topologies, since the hostname label + is sufficient to explicitly identify a proper node. + + topologyAssignment: + levels: + - kubernetes.io/hostname + domains: + - values: [hostname-1] + count: 4 + - values: [hostname-2] + count: 2 + properties: + domains: + description: |- + domains is a list of topology assignments split by topology domains at + the lowest level of the topology. + items: + properties: + count: + description: |- + count indicates the number of Pods to be scheduled in the topology + domain indicated by the values field. + format: int32 + minimum: 1 + type: integer + values: + description: |- + values is an ordered list of node selector values describing a topology + domain. The values correspond to the consecutive topology levels, from + the highest to the lowest. + items: + type: string + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + required: + - count + - values + type: object + type: array + levels: + description: |- + levels is an ordered list of keys denoting the levels of the assigned + topology (i.e. node label keys), from the highest to the lowest level of + the topology. + items: + type: string + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + required: + - domains + - levels + type: object + required: + - name + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - clusterQueue + - podSetAssignments + type: object + admissionChecks: + description: admissionChecks list all the admission checks required + by the workload and the current status + items: + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + name: + description: name identifies the admission check. + maxLength: 316 + type: string + podSetUpdates: + items: + description: |- + PodSetUpdate contains a list of pod set modifications suggested by AdmissionChecks. + The modifications should be additive only - modifications of already existing keys + or having the same key provided by multiple AdmissionChecks is not allowed and will + result in failure during workload admission. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + name: + description: Name of the PodSet to modify. Should match + to one of the Workload's PodSets. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + nodeSelector: + additionalProperties: + type: string + type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + maxItems: 8 + type: array + x-kubernetes-validations: + - message: operator must be Exists when 'key' is empty, + which means 'match all values and all keys' + rule: 'self.all(x, !has(x.key) ? x.operator == ''Exists'' + : true)' + - message: effect must be 'NoExecute' when 'tolerationSeconds' + is set + rule: 'self.all(x, has(x.tolerationSeconds) ? x.effect + == ''NoExecute'' : true)' + - message: 'supported toleration values: ''Equal''(default), + ''Exists''' + rule: self.all(x, !has(x.operator) || x.operator in + ['Equal', 'Exists']) + - message: a value must be empty when 'operator' is 'Exists' + rule: 'self.all(x, has(x.operator) && x.operator == + ''Exists'' ? !has(x.value) : true)' + - message: 'supported taint effect values: ''NoSchedule'', + ''PreferNoSchedule'', ''NoExecute''' + rule: self.all(x, !has(x.effect) || x.effect in ['NoSchedule', + 'PreferNoSchedule', 'NoExecute']) + required: + - name + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + state: + description: state of the admissionCheck, one of Pending, Ready, + Retry, Rejected + enum: + - Pending + - Ready + - Retry + - Rejected + type: string + required: + - lastTransitionTime + - message + - name + - state + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + conditions: + description: |- + conditions hold the latest available observations of the Workload + current state. + + The type of the condition could be: + + - Admitted: the Workload was admitted through a ClusterQueue. + - Finished: the associated workload finished running (failed or succeeded). + - PodsReady: at least `.spec.podSets[*].count` Pods are ready or have + succeeded. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + reclaimablePods: + description: |- + reclaimablePods keeps track of the number pods within a podset for which + the resource reservation is no longer needed. + items: + properties: + count: + description: count is the number of pods for which the requested + resources are no longer needed. + format: int32 + minimum: 0 + type: integer + name: + description: name is the PodSet name. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - count + - name + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + requeueState: + description: |- + requeueState holds the re-queue state + when a workload meets Eviction with PodsReadyTimeout reason. + properties: + count: + description: |- + count records the number of times a workload has been re-queued + When a deactivated (`.spec.activate`=`false`) workload is reactivated (`.spec.activate`=`true`), + this count would be reset to null. + format: int32 + minimum: 0 + type: integer + requeueAt: + description: |- + requeueAt records the time when a workload will be re-queued. + When a deactivated (`.spec.activate`=`false`) workload is reactivated (`.spec.activate`=`true`), + this time would be reset to null. + format: date-time + type: string + type: object + resourceRequests: + description: |- + resourceRequests provides a detailed view of the resources that were + requested by a non-admitted workload when it was considered for admission. + If admission is non-null, resourceRequests will be empty because + admission.resourceUsage contains the detailed information. + items: + properties: + name: + default: main + description: name is the name of the podSet. It should match + one of the names in .spec.podSets. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + resources is the total resources all the pods in the podset need to run. + + Beside what is provided in podSet's specs, this value also takes into account + the LimitRange defaults and RuntimeClass overheads at the moment of consideration + and the application of resource.excludeResourcePrefixes and resource.transformations. + type: object + required: + - name + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + schedulingStats: + description: schedulingStats tracks scheduling statistics + properties: + evictions: + description: evictions tracks eviction statistics by reason and + underlyingCause. + items: + properties: + count: + description: count tracks the number of evictions for this + reason and detailed reason. + format: int32 + minimum: 0 + type: integer + reason: + description: reason specifies the programmatic identifier + for the eviction cause. + maxLength: 316 + type: string + underlyingCause: + description: |- + underlyingCause specifies a finer-grained explanation that complements the eviction reason. + This may be an empty string. + maxLength: 316 + type: string + required: + - count + - reason + - underlyingCause + type: object + type: array + x-kubernetes-list-map-keys: + - reason + - underlyingCause + x-kubernetes-list-type: map + type: object + type: object + type: object + x-kubernetes-validations: + - message: podSetAssignments must have the same number of podSets as the spec + rule: 'has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, + c.type == ''QuotaReserved'' && c.status == ''True'') && has(self.status.admission) + ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) + : true' + - message: field is immutable + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, + c.type == ''QuotaReserved'' && c.status == ''True'')) ? (oldSelf.spec.priorityClassSource + == self.spec.priorityClassSource) : true' + - message: field is immutable + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, + c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassName) + && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName + == self.spec.priorityClassName) : true' + - message: field is immutable + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, + c.type == ''QuotaReserved'' && c.status == ''True'')) && (has(self.status) + && has(self.status.conditions) && self.status.conditions.exists(c, c.type + == ''QuotaReserved'' && c.status == ''True'')) && has(oldSelf.spec.queueName) + && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName + : true' + - message: maximumExecutionTimeSeconds is immutable while admitted + rule: ((has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, + c.type == 'Admitted' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) + && self.status.conditions.exists(c, c.type == 'Admitted' && c.status == + 'True')))?((has(oldSelf.spec.maximumExecutionTimeSeconds)?oldSelf.spec.maximumExecutionTimeSeconds:0) + == (has(self.spec.maximumExecutionTimeSeconds)?self.spec.maximumExecutionTimeSeconds:0)):true + served: true + storage: true + subresources: + status: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-controller-manager + namespace: kueue-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-leader-election-role + namespace: kueue-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.kueue.x-k8s.io/batch-admin: "true" +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-batch-admin-role +--- +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.kueue.x-k8s.io/batch-user: "true" +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-batch-user-role +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-clusterqueue-editor-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-clusterqueue-viewer-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues + verbs: + - get + - list + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-cohort-editor-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - cohorts + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-cohort-viewer-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - cohorts + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-jaxjob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - jaxjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - jaxjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-jaxjob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - jaxjobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - jaxjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-job-editor-role +rules: +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch + resources: + - jobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-job-viewer-role +rules: +- apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch +- apiGroups: + - batch + resources: + - jobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-jobset-editor-role +rules: +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-jobset-viewer-role +rules: +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets + verbs: + - get + - list + - watch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-localqueue-editor-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - localqueues + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - localqueues/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-localqueue-viewer-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - localqueues + verbs: + - get + - list + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - localqueues/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-manager-role +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - watch +- apiGroups: + - "" + resources: + - limitranges + - namespaces + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods/finalizers + verbs: + - get + - update +- apiGroups: + - "" + resources: + - pods/status + verbs: + - get + - patch +- apiGroups: + - "" + resources: + - podtemplates + verbs: + - create + - delete + - get + - list + - update + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - get + - list + - update + - watch +- apiGroups: + - apps + resources: + - deployments + - replicasets + - statefulsets + verbs: + - get + - list + - watch +- apiGroups: + - autoscaling.x-k8s.io + resources: + - provisioningrequests + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - autoscaling.x-k8s.io + resources: + - provisioningrequests/status + verbs: + - get +- apiGroups: + - batch + resources: + - jobs + verbs: + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch + resources: + - jobs/finalizers + - jobs/status + verbs: + - get + - patch + - update +- apiGroups: + - flowcontrol.apiserver.k8s.io + resources: + - flowschemas + - prioritylevelconfigurations + verbs: + - list + - watch +- apiGroups: + - flowcontrol.apiserver.k8s.io + resources: + - flowschemas/status + verbs: + - patch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets + verbs: + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/finalizers + verbs: + - get + - update +- apiGroups: + - jobset.x-k8s.io + resources: + - jobsets/status + verbs: + - get + - patch + - update +- apiGroups: + - kubeflow.org + resources: + - jaxjobs + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - jaxjobs/finalizers + - mpijobs/finalizers + - paddlejobs/finalizers + - pytorchjobs/finalizers + - tfjobs/finalizers + - xgboostjobs/finalizers + verbs: + - get + - update +- apiGroups: + - kubeflow.org + resources: + - jaxjobs/status + - mpijobs/status + - paddlejobs/status + - pytorchjobs/status + - tfjobs/status + - xgboostjobs/status + verbs: + - get + - patch + - update +- apiGroups: + - kubeflow.org + resources: + - mpijobs + - paddlejobs + - pytorchjobs + - tfjobs + - xgboostjobs + verbs: + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - admissionchecks + - clusterqueues + - cohorts + - localqueues + - workloads + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - admissionchecks/finalizers + - clusterqueues/finalizers + - localqueues/finalizers + - resourceflavors/finalizers + - topologies/finalizers + - workloads/finalizers + verbs: + - update +- apiGroups: + - kueue.x-k8s.io + resources: + - admissionchecks/status + - clusterqueues/status + - cohorts/status + - localqueues/status + - multikueueclusters/status + - workloads/status + verbs: + - get + - patch + - update +- apiGroups: + - kueue.x-k8s.io + resources: + - multikueueclusters + - multikueueconfigs + - provisioningrequestconfigs + - workloadpriorityclasses + verbs: + - get + - list + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - resourceflavors + verbs: + - delete + - get + - list + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - topologies + verbs: + - get + - list + - update + - watch +- apiGroups: + - leaderworkerset.x-k8s.io + resources: + - leaderworkersets + verbs: + - get + - list + - watch +- apiGroups: + - node.k8s.io + resources: + - runtimeclasses + verbs: + - get + - list + - watch +- apiGroups: + - ray.io + resources: + - rayclusters + - rayjobs + verbs: + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ray.io + resources: + - rayclusters/finalizers + - rayjobs/finalizers + verbs: + - get + - update +- apiGroups: + - ray.io + resources: + - rayclusters/status + - rayjobs/status + verbs: + - get + - patch + - update +- apiGroups: + - scheduling.k8s.io + resources: + - priorityclasses + verbs: + - get + - list + - watch +- apiGroups: + - workload.codeflare.dev + resources: + - appwrappers + verbs: + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - workload.codeflare.dev + resources: + - appwrappers/finalizers + verbs: + - get + - update +- apiGroups: + - workload.codeflare.dev + resources: + - appwrappers/status + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-mpijob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - mpijobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - mpijobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-mpijob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - mpijobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - mpijobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-paddlejob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - paddlejobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - paddlejobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-paddlejob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - paddlejobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - paddlejobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-pending-workloads-cq-viewer-role +rules: +- apiGroups: + - visibility.kueue.x-k8s.io + resources: + - clusterqueues/pendingworkloads + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-pending-workloads-lq-viewer-role +rules: +- apiGroups: + - visibility.kueue.x-k8s.io + resources: + - localqueues/pendingworkloads + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-pytorchjob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-pytorchjob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-raycluster-editor-role +rules: +- apiGroups: + - ray.io + resources: + - rayclusters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ray.io + resources: + - rayclusters/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-raycluster-viewer-role +rules: +- apiGroups: + - ray.io + resources: + - rayclusters + verbs: + - get + - list + - watch +- apiGroups: + - ray.io + resources: + - rayclusters/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-rayjob-editor-role +rules: +- apiGroups: + - ray.io + resources: + - rayjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ray.io + resources: + - rayjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-rayjob-viewer-role +rules: +- apiGroups: + - ray.io + resources: + - rayjobs + verbs: + - get + - list + - watch +- apiGroups: + - ray.io + resources: + - rayjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-resourceflavor-editor-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - resourceflavors + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-resourceflavor-viewer-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - resourceflavors + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-tfjob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - tfjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - tfjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-tfjob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - tfjobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - tfjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-topology-editor-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - topologies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-topology-viewer-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - topologies + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + name: kueue-workload-editor-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - workloads + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - workloads/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-workload-viewer-role +rules: +- apiGroups: + - kueue.x-k8s.io + resources: + - workloads + verbs: + - get + - list + - watch +- apiGroups: + - kueue.x-k8s.io + resources: + - workloads/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-xgboostjob-editor-role +rules: +- apiGroups: + - kubeflow.org + resources: + - xgboostjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubeflow.org + resources: + - xgboostjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + rbac.kueue.x-k8s.io/batch-admin: "true" + rbac.kueue.x-k8s.io/batch-user: "true" + name: kueue-xgboostjob-viewer-role +rules: +- apiGroups: + - kubeflow.org + resources: + - xgboostjobs + verbs: + - get + - list + - watch +- apiGroups: + - kubeflow.org + resources: + - xgboostjobs/status + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-visibility-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: kueue-controller-manager + namespace: kueue-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-leader-election-rolebinding + namespace: kueue-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kueue-leader-election-role +subjects: +- kind: ServiceAccount + name: kueue-controller-manager + namespace: kueue-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kueue-manager-role +subjects: +- kind: ServiceAccount + name: kueue-controller-manager + namespace: kueue-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kueue-metrics-auth-role +subjects: +- kind: ServiceAccount + name: kueue-controller-manager + namespace: kueue-system +--- +apiVersion: v1 +data: + controller_manager_config.yaml: | + apiVersion: config.kueue.x-k8s.io/v1beta1 + kind: Configuration + health: + healthProbeBindAddress: :8081 + metrics: + bindAddress: :8443 + # enableClusterQueueResources: true + webhook: + port: 9443 + leaderElection: + leaderElect: true + resourceName: c1f6bfd2.kueue.x-k8s.io + controller: + groupKindConcurrency: + Job.batch: 5 + Pod: 5 + Workload.kueue.x-k8s.io: 5 + LocalQueue.kueue.x-k8s.io: 1 + Cohort.kueue.x-k8s.io: 1 + ClusterQueue.kueue.x-k8s.io: 1 + ResourceFlavor.kueue.x-k8s.io: 1 + clientConnection: + qps: 50 + burst: 100 + #pprofBindAddress: :8083 + #waitForPodsReady: + # enable: false + # timeout: 5m + # recoveryTimeout: 3m + # blockAdmission: false + # requeuingStrategy: + # timestamp: Eviction + # backoffLimitCount: null # null indicates infinite requeuing + # backoffBaseSeconds: 60 + # backoffMaxSeconds: 3600 + #manageJobsWithoutQueueName: true + #managedJobsNamespaceSelector: + # matchExpressions: + # - key: kubernetes.io/metadata.name + # operator: NotIn + # values: [ kube-system, kueue-system ] + #internalCertManagement: + # enable: false + # webhookServiceName: "" + # webhookSecretName: "" + integrations: + frameworks: + - "batch/job" + - "kubeflow.org/mpijob" + - "ray.io/rayjob" + - "ray.io/raycluster" + - "jobset.x-k8s.io/jobset" + - "kubeflow.org/paddlejob" + - "kubeflow.org/pytorchjob" + - "kubeflow.org/tfjob" + - "kubeflow.org/xgboostjob" + - "kubeflow.org/jaxjob" + - "workload.codeflare.dev/appwrapper" + # - "pod" + # - "deployment" # requires enabling pod integration + # - "statefulset" # requires enabling pod integration + # - "leaderworkerset.x-k8s.io/leaderworkerset" # requires enabling pod integration + # externalFrameworks: + # - "Foo.v1.example.com" + #fairSharing: + # enable: true + # preemptionStrategies: [LessThanOrEqualToFinalShare, LessThanInitialShare] + #admissionFairSharing: + # usageHalfLifeTime: "168h" # 7 days + # usageSamplingInterval: "5m" + # resourceWeights: # optional, defaults to 1 for all resources if not specified + # cpu: 0 # if you want to completely ignore cpu usage + # memory: 0 # ignore completely memory usage + # example.com/gpu: 100 # and you care only about GPUs usage + #resources: + # excludeResourcePrefixes: [] + # transformations: + # - input: nvidia.com/mig-4g.5gb + # strategy: Replace | Retain + # outputs: + # example.com/accelerator-memory: 5Gi + # example.com/accelerator-gpc: 4 + #objectRetentionPolicies: + # workloads: + # afterFinished: null # null indicates infinite retention, 0s means no retention at all + # afterDeactivatedByKueue: null # null indicates infinite retention, 0s means no retention at all +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-manager-config + namespace: kueue-system +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-webhook-server-cert + namespace: kueue-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-controller-manager-metrics-service + namespace: kueue-system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + control-plane: controller-manager +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-visibility-server + namespace: kueue-system +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: 8082 + selector: + control-plane: controller-manager +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-webhook-service + namespace: kueue-system +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-controller-manager + namespace: kueue-system +spec: + replicas: 1 + selector: + matchLabels: + control-plane: controller-manager + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + spec: + containers: + - args: + - --config=/controller_manager_config.yaml + - --zap-log-level=2 + - --feature-gates=TopologyAwareScheduling=true + command: + - /manager + image: registry.k8s.io/kueue/kueue:v0.12.2 + imagePullPolicy: Always + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: manager + ports: + - containerPort: 8443 + name: metrics + protocol: TCP + - containerPort: 8082 + name: visibility + protocol: TCP + - containerPort: 9443 + name: webhook-server + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: "2" + memory: 512Mi + requests: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /visibility + name: visibility + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + - mountPath: /controller_manager_config.yaml + name: manager-config + subPath: controller_manager_config.yaml + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + serviceAccountName: kueue-controller-manager + terminationGracePeriodSeconds: 10 + volumes: + - emptyDir: {} + name: visibility + - name: cert + secret: + defaultMode: 420 + secretName: kueue-webhook-server-cert + - configMap: + name: kueue-manager-config + name: manager-config + tolerations: + - effect: NoSchedule + key: components.gke.io/gke-managed-components + operator: Equal + value: "true" +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: v1beta1.visibility.kueue.x-k8s.io +spec: + group: visibility.kueue.x-k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: kueue-visibility-server + namespace: kueue-system + version: v1beta1 + versionPriority: 100 +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate--v1-pod + failurePolicy: Fail + name: mpod.kb.io + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kueue-system + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-apps-v1-deployment + failurePolicy: Fail + name: mdeployment.kb.io + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kueue-system + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - deployments + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-apps-v1-statefulset + failurePolicy: Fail + name: mstatefulset.kb.io + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kueue-system + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - statefulsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-workload-codeflare-dev-v1beta2-appwrapper + failurePolicy: Fail + name: mappwrapper.kb.io + rules: + - apiGroups: + - workload.codeflare.dev + apiVersions: + - v1beta2 + operations: + - CREATE + resources: + - appwrappers + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-batch-v1-job + failurePolicy: Fail + name: mjob.kb.io + rules: + - apiGroups: + - batch + apiVersions: + - v1 + operations: + - CREATE + resources: + - jobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-jobset-x-k8s-io-v1alpha2-jobset + failurePolicy: Fail + name: mjobset.kb.io + rules: + - apiGroups: + - jobset.x-k8s.io + apiVersions: + - v1alpha2 + operations: + - CREATE + resources: + - jobsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v1-jaxjob + failurePolicy: Fail + name: mjaxjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + resources: + - jaxjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v1-paddlejob + failurePolicy: Fail + name: mpaddlejob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + resources: + - paddlejobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v1-pytorchjob + failurePolicy: Fail + name: mpytorchjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + resources: + - pytorchjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v1-tfjob + failurePolicy: Fail + name: mtfjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + resources: + - tfjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v1-xgboostjob + failurePolicy: Fail + name: mxgboostjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + resources: + - xgboostjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-leaderworkerset-x-k8s-io-v1-leaderworkerset + failurePolicy: Fail + name: mleaderworkerset.kb.io + rules: + - apiGroups: + - leaderworkerset.x-k8s.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - leaderworkersets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kubeflow-org-v2beta1-mpijob + failurePolicy: Fail + name: mmpijob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v2beta1 + operations: + - CREATE + resources: + - mpijobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-ray-io-v1-raycluster + failurePolicy: Fail + name: mraycluster.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + resources: + - rayclusters + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-ray-io-v1-rayjob + failurePolicy: Fail + name: mrayjob.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + resources: + - rayjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kueue-x-k8s-io-v1beta1-clusterqueue + failurePolicy: Fail + name: mclusterqueue.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + resources: + - clusterqueues + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kueue-x-k8s-io-v1beta1-resourceflavor + failurePolicy: Fail + name: mresourceflavor.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + resources: + - resourceflavors + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /mutate-kueue-x-k8s-io-v1beta1-workload + failurePolicy: Fail + name: mworkload.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + resources: + - workloads + sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: kueue + control-plane: controller-manager + name: kueue-validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate--v1-pod + failurePolicy: Fail + name: vpod.kb.io + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kueue-system + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - pods + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-apps-v1-deployment + failurePolicy: Fail + name: vdeployment.kb.io + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kueue-system + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - deployments + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-apps-v1-statefulset + failurePolicy: Fail + name: vstatefulset.kb.io + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - kueue-system + rules: + - apiGroups: + - apps + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - statefulsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-workload-codeflare-dev-v1beta2-appwrapper + failurePolicy: Fail + name: vappwrapper.kb.io + rules: + - apiGroups: + - workload.codeflare.dev + apiVersions: + - v1beta2 + operations: + - CREATE + - UPDATE + resources: + - appwrappers + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-batch-v1-job + failurePolicy: Fail + name: vjob.kb.io + rules: + - apiGroups: + - batch + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - jobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-jobset-x-k8s-io-v1alpha2-jobset + failurePolicy: Fail + name: vjobset.kb.io + rules: + - apiGroups: + - jobset.x-k8s.io + apiVersions: + - v1alpha2 + operations: + - CREATE + - UPDATE + resources: + - jobsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v1-jaxjob + failurePolicy: Fail + name: vjaxjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - jaxjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v1-paddlejob + failurePolicy: Fail + name: vpaddlejob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - paddlejobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v1-pytorchjob + failurePolicy: Fail + name: vpytorchjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - pytorchjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v1-tfjob + failurePolicy: Fail + name: vtfjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - tfjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v1-xgboostjob + failurePolicy: Fail + name: vxgboostjob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - xgboostjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-leaderworkerset-x-k8s-io-v1-leaderworkerset + failurePolicy: Fail + name: vleaderworkerset.kb.io + rules: + - apiGroups: + - leaderworkerset.x-k8s.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - leaderworkersets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kubeflow-org-v2beta1-mpijob + failurePolicy: Fail + name: vmpijob.kb.io + rules: + - apiGroups: + - kubeflow.org + apiVersions: + - v2beta1 + operations: + - CREATE + - UPDATE + resources: + - mpijobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-ray-io-v1-raycluster + failurePolicy: Fail + name: vraycluster.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - rayclusters + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-ray-io-v1-rayjob + failurePolicy: Fail + name: vrayjob.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - rayjobs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kueue-x-k8s-io-v1beta1-clusterqueue + failurePolicy: Fail + name: vclusterqueue.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - clusterqueues + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kueue-x-k8s-io-v1alpha1-cohort + failurePolicy: Fail + name: vcohort.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - cohorts + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kueue-x-k8s-io-v1beta1-resourceflavor + failurePolicy: Fail + name: vresourceflavor.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - resourceflavors + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: kueue-webhook-service + namespace: kueue-system + path: /validate-kueue-x-k8s-io-v1beta1-workload + failurePolicy: Fail + name: vworkload.kb.io + rules: + - apiGroups: + - kueue.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - workloads + - workloads/status + sideEffects: None diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/metadata.yaml new file mode 100644 index 0000000000..17bedb471b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/providers.tf b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/providers.tf new file mode 100644 index 0000000000..ba9a4af370 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/providers.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +provider "kubectl" { + host = "https://${data.google_container_cluster.gke_cluster.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(data.google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate) + load_config_file = false + apply_retry_count = 15 # Terraform may apply resources in parallel, leading to potential dependency issues. This retry mechanism ensures that if a resource's dependencies aren't ready, Terraform will attempt to apply it again. +} + +provider "helm" { + kubernetes { + host = "https://${data.google_container_cluster.gke_cluster.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode( + data.google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate, + ) + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/variables.tf new file mode 100644 index 0000000000..61bed61d4f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/variables.tf @@ -0,0 +1,166 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + kueue_supported_versions = ["v0.12.2", "v0.11.4", "v0.10.1", "v0.10.0"] + jobset_supported_versions = ["v0.8.1", "v0.7.2", "v0.5.2"] + gib_supported_versions = ["v1.0.2", "v1.0.3", "v1.0.5", "v1.0.6"] +} + +resource "terraform_data" "kueue_validations" { + lifecycle { + precondition { + condition = !var.kueue.install || contains(local.kueue_supported_versions, var.kueue.version) + error_message = "Supported version of Kueue are ${join(", ", local.kueue_supported_versions)}" + } + } +} + +resource "terraform_data" "jobset_validations" { + lifecycle { + precondition { + condition = !var.jobset.install || contains(local.jobset_supported_versions, var.jobset.version) + error_message = "Supported version of Jobset are ${join(", ", local.jobset_supported_versions)}" + } + } +} + +resource "terraform_data" "gib_validations" { + lifecycle { + precondition { + condition = !var.gib.install || contains(local.gib_supported_versions, var.gib.template_vars.version) + error_message = "Supported version of the NCCL gIB plugin are ${join(", ", local.gib_supported_versions)}" + } + } +} + +resource "terraform_data" "initial_gib_version" { + input = var.gib.install ? var.gib.template_vars.version : null + + lifecycle { + ignore_changes = [input] + } +} + +check "gib_version_changes" { + assert { + # Skip version checking if gIB was not initially or is not currently installed + condition = terraform_data.initial_gib_version.output == null || !var.gib.install || terraform_data.initial_gib_version.output == var.gib.template_vars.version + error_message = "When changing the gIB NCCL plugin version, confirm full rollout and environment consistency. Replace any NCCL env hard coding/caches with set_nccl_env.sh sourcing." + } +} + +variable "project_id" { + description = "The project ID that hosts the gke cluster." + type = string +} + +variable "cluster_id" { + description = "An identifier for the gke cluster resource with format projects//locations//clusters/." + type = string + nullable = false +} + +variable "apply_manifests" { + description = "A list of manifests to apply to GKE cluster using kubectl. For more details see [kubectl module's inputs](kubectl/README.md)." + type = list(object({ + enable = optional(bool, true) + content = optional(string, null) + source = optional(string, null) + template_vars = optional(map(any), null) + server_side_apply = optional(bool, false) + wait_for_rollout = optional(bool, true) + })) + default = [] +} + + +variable "kueue" { + description = "Install and configure [Kueue](https://kueue.sigs.k8s.io/docs/overview/) workload scheduler. A configuration yaml/template file can be provided with config_path to be applied right after kueue installation. If a template file provided, its variables can be set to config_template_vars." + type = object({ + install = optional(bool, false) + version = optional(string, "v0.12.2") + config_path = optional(string, null) + config_template_vars = optional(map(any), null) + }) + default = {} +} + +variable "gke_cluster_exists" { + description = "A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations." + type = bool + default = false +} + +variable "jobset" { + description = "Install [Jobset](https://github.com/kubernetes-sigs/jobset) which manages a group of K8s [jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) as a unit." + type = object({ + install = optional(bool, false) + version = optional(string, "v0.7.2") + }) + default = {} +} + + +variable "gpu_operator" { + description = "Install [GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) which uses the [Kubernetes operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) to automate the management of all NVIDIA software components needed to provision GPU." + type = object({ + install = optional(bool, false) + version = optional(string, "v25.3.0") + }) + default = {} +} + +variable "nvidia_dra_driver" { + description = "Installs [Nvidia DRA driver](https://github.com/NVIDIA/k8s-dra-driver-gpu) which supports Dynamic Resource Allocation for NVIDIA GPUs in Kubernetes" + type = object({ + install = optional(bool, false) + version = optional(string, "v25.3.0-rc.3") + }) + default = {} +} + +variable "gib" { + description = "Install the NCCL gIB plugin" + type = object({ + install = bool + path = string + template_vars = object({ + image = optional(string, "us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib") + version = string + node_affinity = optional(any, { + requiredDuringSchedulingIgnoredDuringExecution = { + nodeSelectorTerms = [{ + matchExpressions = [{ + key = "cloud.google.com/gke-gpu", + operator = "In", + values = ["true"] + }] + }] + } + }) + accelerator_count = number + }) + }) + default = { + install = false + path = "" + template_vars = { + version = "" + accelerator_count = 0 + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/versions.tf new file mode 100644 index 0000000000..e5cc65acb1 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/versions.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "> 5.0" + } + kubectl = { + source = "gavinbunney/kubectl" + version = ">= 1.7.0" + } + helm = { + source = "hashicorp/helm" + version = "~> 2.17" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:kubectl-apply/v1.37.2" + } + + required_version = ">= 1.3" +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/README.md b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/README.md new file mode 100644 index 0000000000..949346525e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/README.md @@ -0,0 +1,86 @@ +## Description + +Creates a [monitoring dashboard][gcp-dash] for the HPC cluster deployment. The +module includes a default HPC-focused dashboard with the ability to add custom +widgets as well as the option to add an empty dashboard and add widgets as +needed. + +[gcp-dash]: https://cloud.google.com/monitoring/charts/predefined-dashboards + +## Example + +```yaml +- id: hpc_dash + source: modules/monitoring/dashboard + settings: + widgets: + - | + { + "text": { + "content": "## Header", + "format": "MARKDOWN" + }, + "title": "Custom Text Block Widget" + } +``` + +This module creates a dashboard based on the HPC dashboard (default) with an +extra text widget added as a multi-line string representing a JSON block. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_monitoring_dashboard.dashboard](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_dashboard) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [base\_dashboard](#input\_base\_dashboard) | Baseline dashboard template, select from HPC or Empty | `string` | `"HPC"` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the monitoring dashboard instance. Key-value pairs. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [title](#input\_title) | Title of the created dashboard | `string` | `"Cluster Toolkit Dashboard"` | no | +| [widgets](#input\_widgets) | List of additional widgets to add to the base dashboard. | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Instructions for accessing the monitoring dashboard | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl new file mode 100644 index 0000000000..f25cbbd2c6 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl @@ -0,0 +1,17 @@ +{ + "displayName": "${title}: ${deployment_name}", + "gridLayout": { + "columns": 2, + "widgets": [ + { + "text": { + "content": "Metrics from the ${deployment_name} deployment of the Cluster Toolkit.", + "format": "MARKDOWN" + }, + "title": "${title}" + }%{ for widget in widgets ~}, + ${widget} + %{endfor ~} + ] + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl new file mode 100644 index 0000000000..5b20435a9a --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl @@ -0,0 +1,595 @@ +{ + "displayName": "${title}: ${deployment_name}", + "labels": ${jsonencode(labels)}, + "gridLayout": { + "columns": 2, + "widgets": [ + { + "text": { + "content": "HPC metrics from the ${deployment_name} deployment of the Cluster Toolkit.", + "format": "MARKDOWN" + }, + "title": "${title}" + }, + { + "title": "VM Instance - Memory utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"agent.googleapis.com/memory/percent_used\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - CPU Utilization", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"", + "pickTimeSeriesFilter": { + "direction": "TOP", + "numTimeSeries": 20, + "rankingMethod": "METHOD_MEAN" + } + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - CPU utilization (agent)", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"agent.googleapis.com/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + }, + "unitOverride": "%" + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Disk read operations", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/disk/read_ops_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Disk write operations", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/disk/write_ops_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Disk Read Bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"agent.googleapis.com/disk/read_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Disk Write Bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"agent.googleapis.com/disk/write_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "Throttled read bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/disk/throttled_read_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "Throttled write bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/disk/throttled_write_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Received packets", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/network/received_packets_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "VM Instance - Sent packets", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/network/sent_packets_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "VM Instance - Received bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/network/received_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Sent bytes", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_MEAN", + "groupByFields": [ + "metric.label.\"instance_name\"", + "metric.label.\"loadbalanced\"", + "resource.label.\"project_id\"", + "resource.label.\"instance_id\"", + "resource.label.\"zone\"" + ], + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/network/sent_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"", + "secondaryAggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + } + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "GCE VM Instance - Network Traffic Bytes (agent)", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"agent.googleapis.com/interface/traffic\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "Network Packets (agent)", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_RATE" + }, + "filter": "metric.type=\"agent.googleapis.com/interface/packets\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "TCP connections", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"agent.googleapis.com/network/tcp_connections\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + }, + "unitOverride": "1" + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "VM Instance - CPU utilization for steal", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "STACKED_BAR", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MAX" + }, + "filter": "metric.type=\"agent.googleapis.com/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\" metric.label.\"cpu_state\"=\"steal\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + { + "title": "VM Instance - CPU utilization [MEAN]", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "60s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "apiSource": "DEFAULT_CLOUD", + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "60s", + "crossSeriesReducer": "REDUCE_NONE", + "perSeriesAligner": "ALIGN_MEAN" + }, + "filter": "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }%{ for widget in widgets ~}, + ${widget} + %{endfor ~} + ] + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/main.tf b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/main.tf new file mode 100644 index 0000000000..df3c5c36b0 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/main.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "dashboard", ghpc_role = "monitoring" }) +} + +locals { + dash_path = "${path.module}/dashboards/${var.base_dashboard}.json.tpl" +} + +resource "google_monitoring_dashboard" "dashboard" { + dashboard_json = templatefile(local.dash_path, { + widgets = var.widgets + deployment_name = var.deployment_name + title = var.title + labels = local.labels + } + ) + project = var.project_id +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/metadata.yaml new file mode 100644 index 0000000000..de1a10f57d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - stackdriver.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/outputs.tf new file mode 100644 index 0000000000..b7ff35fb0e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/outputs.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "instructions" { + description = "Instructions for accessing the monitoring dashboard" + value = <<-EOT + A monitoring dashboard has been created. To view, navigate to the following URL: + https://console.cloud.google.com/monitoring/dashboards/builder${regex("/[0-9a-z-]*$", google_monitoring_dashboard.dashboard.id)} + EOT +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/variables.tf new file mode 100644 index 0000000000..8194f8b73a --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/variables.tf @@ -0,0 +1,52 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "base_dashboard" { + description = "Baseline dashboard template, select from HPC or Empty" + type = string + default = "HPC" + validation { + condition = contains(["HPC", "Empty"], var.base_dashboard) + error_message = "Must set var.base_dashboard to either \"HPC\" or \"Empty\"." + } +} + +variable "title" { + description = "Title of the created dashboard" + type = string + default = "Cluster Toolkit Dashboard" +} + +variable "widgets" { + description = "List of additional widgets to add to the base dashboard." + type = list(string) + default = [] +} + +variable "labels" { + description = "Labels to add to the monitoring dashboard instance. Key-value pairs." + type = map(string) +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/versions.tf new file mode 100644 index 0000000000..d8c4587051 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:dashboard/v1.57.0" + } + + required_version = ">= 0.14.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/README.md b/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/README.md new file mode 100644 index 0000000000..b17058daf4 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/README.md @@ -0,0 +1,107 @@ +## Description + +This module facilitates the creation of custom firewall rules for existing +networks. + +## Example usage + +This module can be used by other Toolkit modules to create application-specific +firewall rules or in conjunction with the [pre-existing-vpc] module to enable +traffic in existing networks. The snippet below is drawn from the +[ml-slurm.yaml] example: + +```yaml +- group: primary + modules: + - id: network + source: modules/network/pre-existing-vpc + + # this example anticipates that the VPC default network has internal traffic + # allowed and IAP tunneling for SSH connections + - id: firewall_rule + source: modules/network/firewall-rules + use: + - network + settings: + ingress_rules: + - name: $(vars.deployment_name)-allow-internal-traffic + description: Allow internal traffic + destination_ranges: + - $(network.subnetwork_address) + source_ranges: + - $(network.subnetwork_address) + allow: + - protocol: tcp + ports: + - 0-65535 + - protocol: udp + ports: + - 0-65535 + - protocol: icmp + - name: $(vars.deployment_name)-allow-iap-ssh + description: Allow IAP-tunneled SSH connections + destination_ranges: + - $(network.subnetwork_address) + source_ranges: + - 35.235.240.0/20 + allow: + - protocol: tcp + ports: + - 22 +``` + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [firewall\_rule](#module\_firewall\_rule) | terraform-google-modules/network/google//modules/firewall-rules | ~> 9.0 | + +## Resources + +| Name | Type | +|------|------| +| [google_compute_subnetwork.subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [egress\_rules](#input\_egress\_rules) | List of egress rules | 
list(object({
    name                    = string
    description             = optional(string, null)
    disabled                = optional(bool, null)
    priority                = optional(number, null)
    destination_ranges      = optional(list(string), [])
    source_ranges           = optional(list(string), [])
    source_tags             = optional(list(string))
    source_service_accounts = optional(list(string))
    target_tags             = optional(list(string))
    target_service_accounts = optional(list(string))

    allow = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    deny = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    log_config = optional(object({
      metadata = string
    }))
  }))
| `[]` | no | +| [ingress\_rules](#input\_ingress\_rules) | List of ingress rules | 
list(object({
    name                    = string
    description             = optional(string, null)
    disabled                = optional(bool, null)
    priority                = optional(number, null)
    destination_ranges      = optional(list(string), [])
    source_ranges           = optional(list(string), [])
    source_tags             = optional(list(string))
    source_service_accounts = optional(list(string))
    target_tags             = optional(list(string))
    target_service_accounts = optional(list(string))

    allow = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    deny = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    log_config = optional(object({
      metadata = string
    }))
  }))
| `[]` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork whose global network firewall rules will be modified. | `string` | n/a | yes | + +## Outputs + +No outputs. + + +[pre-existing-vpc]: ../pre-existing-vpc/README.md +[ml-slurm.yaml]: ../../../examples/ml-slurm.yaml diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/main.tf b/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/main.tf new file mode 100644 index 0000000000..f3f24e78d3 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/main.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# the google_compute_network data source does not allow identification by +# self_link, which uniquely identifies subnet, project, and network +data "google_compute_subnetwork" "subnetwork" { + self_link = var.subnetwork_self_link +} + +# Module-level check for Private Google Access on the subnetwork +check "private_google_access_enabled_subnetwork" { + assert { + condition = data.google_compute_subnetwork.subnetwork.private_ip_google_access + error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.subnetwork.name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." + } +} + +module "firewall_rule" { + source = "terraform-google-modules/network/google//modules/firewall-rules" + version = "~> 9.0" + project_id = data.google_compute_subnetwork.subnetwork.project + network_name = data.google_compute_subnetwork.subnetwork.network + + ingress_rules = var.ingress_rules + egress_rules = var.egress_rules +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/variables.tf new file mode 100644 index 0000000000..e7bea8322e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/variables.tf @@ -0,0 +1,76 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork whose global network firewall rules will be modified." + type = string +} + +variable "ingress_rules" { + description = "List of ingress rules" + default = [] + type = list(object({ + name = string + description = optional(string, null) + disabled = optional(bool, null) + priority = optional(number, null) + destination_ranges = optional(list(string), []) + source_ranges = optional(list(string), []) + source_tags = optional(list(string)) + source_service_accounts = optional(list(string)) + target_tags = optional(list(string)) + target_service_accounts = optional(list(string)) + + allow = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + deny = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + log_config = optional(object({ + metadata = string + })) + })) +} + +variable "egress_rules" { + description = "List of egress rules" + default = [] + type = list(object({ + name = string + description = optional(string, null) + disabled = optional(bool, null) + priority = optional(number, null) + destination_ranges = optional(list(string), []) + source_ranges = optional(list(string), []) + source_tags = optional(list(string)) + source_service_accounts = optional(list(string)) + target_tags = optional(list(string)) + target_service_accounts = optional(list(string)) + + allow = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + deny = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + log_config = optional(object({ + metadata = string + })) + })) +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/versions.tf new file mode 100644 index 0000000000..c892bc0435 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:firewall-rules/v1.57.0" + } + + required_version = ">= 1.5" +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/README.md b/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/README.md new file mode 100644 index 0000000000..24dc606c88 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/README.md @@ -0,0 +1,143 @@ +## Description + +This module accomplishes the following: + +* Creates one [VPC network][cft-network] + * Each VPC contains a variable number of subnetworks as specified in the + `subnetworks_template` variable + * Each subnetwork contains distinct IP address ranges +* Outputs the following unique parameters + * `subnetwork_interfaces` which is compatible with Slurm and vm-instance + modules + * `subnetwork_interfaces_gke` which is compatible with GKE modules + +This module is a simplified version of the VPC module and its main difference +is the variable `subnetwork_template` which is the template for all subnetworks +created within the network. This template contains the following values: + +1. `count`: The number of subnetworks to be created +1. `name_prefix`: The prefix for the subnetwork names +1. `ip_range`: [CIDR-formatted IP range][cidr] +1. `region`: The region where the subnetwork will be deployed + +> [!WARNING] +> The `ip_range` should be always be large enough to split into `count` +> subnetworks and the number of required connections within. + +[cft-network]: https://github.com/terraform-google-modules/terraform-google-network/tree/v10.0.0 +[cidr]: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation + +### Example + +This snippet uses the gpu-vpc module to create a new VPC network named +`test-rdma-net` with 8 subnetworks named `test-mrdma-sub-#` where # ranges from +0 to 7. The subnetworks will split the `ip_range` evenly, starting from bit 16 +(0 indexed). The networks are ingested by the Slurm nodeset within the +`additional_networks` setting. + +```yaml + - id: rdma-net + source: modules/network/gpu-rdma-vpc + settings: + network_name: test-rdma-net + network_profile: https://www.googleapis.com/compute/beta/projects/$(vars.project_id)/global/networkProfiles/$(vars.zone)-vpc-roce + network_routing_mode: REGIONAL + subnetworks_template: + name_prefix: test-mrdma-sub + count: 8 + ip_range: 192.168.0.0/16 + region: $(vars.region) + + - id: a3_nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [network0] + settings: + machine_type: a3-ultragpu-8g + additional_networks: + $(concat( + [{ + network=null, + subnetwork=network1.subnetwork_self_link, + subnetwork_project=vars.project_id, + nic_type="GVNIC", + queue_count=null, + network_ip="", + stack_type=null, + access_config=[], + ipv6_access_config=[], + alias_ip_range=[] + }], + rdma-net.subnetwork_interfaces + )) + ... +``` + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [vpc](#module\_vpc) | terraform-google-modules/network/google | ~> 10.0 | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | DEPRECATED: enable\_internal\_traffic can not be specified for gpu-rdma-vpc. | `bool` | `null` | no | +| [firewall\_log\_config](#input\_firewall\_log\_config) | DEPRECATED: firewall\_log\_config can not be specified for gpu-rdma-vpc. | `string` | `null` | no | +| [firewall\_rules](#input\_firewall\_rules) | DEPRECATED: firewall\_rules can not be specified for gpu-rdma-vpc. | `any` | `null` | no | +| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | +| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | +| [network\_name](#input\_network\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-net") | `string` | `null` | no | +| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}} | `string` | n/a | yes | +| [network\_routing\_mode](#input\_network\_routing\_mode) | The network routing mode (default "REGIONAL") | `string` | `"REGIONAL"` | no | +| [nic\_type](#input\_nic\_type) | NIC type for use in modules that use the output | `string` | `"MRDMA"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | +| [shared\_vpc\_host](#input\_shared\_vpc\_host) | Makes this project a Shared VPC host if 'true' (default 'false') | `bool` | `false` | no | +| [subnetworks\_template](#input\_subnetworks\_template) | Specifications for the subnetworks that will be created within this VPC.

count (number, required, number of subnets to create, default is 8)
name\_prefix (string, required, subnet name prefix, default is deployment name)
ip\_range (string, required, range of IPs for all subnets to share (CIDR format), default is 192.168.0.0/16)
region (string, optional, region to deploy subnets to, defaults to vars.region) | 
object({
    count       = number
    name_prefix = string
    ip_range    = string
    region      = optional(string)
  })
| 
{
  "count": 8,
  "ip_range": "192.168.0.0/16",
  "name_prefix": null,
  "region": null
}
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| [network\_id](#output\_network\_id) | ID of the new VPC network | +| [network\_name](#output\_network\_name) | Name of the new VPC network | +| [network\_self\_link](#output\_network\_self\_link) | Self link of the new VPC network | +| [subnetwork\_interfaces](#output\_subnetwork\_interfaces) | Full list of subnetwork objects belonging to the new VPC network (compatible with vm-instance and Slurm modules) | +| [subnetwork\_interfaces\_gke](#output\_subnetwork\_interfaces\_gke) | Full list of subnetwork objects belonging to the new VPC network (compatible with gke-node-pool) | +| [subnetwork\_name\_prefix](#output\_subnetwork\_name\_prefix) | Prefix of the RDMA subnetwork names | +| [subnetworks](#output\_subnetworks) | Full list of subnetwork objects belonging to the new VPC network | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/main.tf b/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/main.tf new file mode 100644 index 0000000000..7f2a141db5 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/main.tf @@ -0,0 +1,79 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + autoname = replace(var.deployment_name, "_", "-") + network_name = var.network_name == null ? "${local.autoname}-net" : var.network_name + subnet_prefix = var.subnetworks_template.name_prefix == null ? "${local.autoname}-subnet" : var.subnetworks_template.name_prefix + + new_bits = ceil(log(var.subnetworks_template.count, 2)) + template_subnetworks = [for i in range(var.subnetworks_template.count) : + { + subnet_name = "${local.subnet_prefix}-${i}" + subnet_region = try(var.subnetworks_template.region, var.region) + subnet_ip = cidrsubnet(var.subnetworks_template.ip_range, local.new_bits, i) + } + ] + + firewall_rules = [] + + output_subnets = [ + for subnet in module.vpc.subnets : { + network = null + subnetwork = subnet.self_link + subnetwork_project = null # will populate from subnetwork_self_link + network_ip = null + nic_type = var.nic_type + stack_type = null + queue_count = null + access_config = [] + ipv6_access_config = [] + alias_ip_range = [] + } + ] + + output_subnets_gke = [ + for i in range(length(module.vpc.subnets)) : { + network = local.network_name + subnetwork = local.template_subnetworks[i].subnet_name + subnetwork_project = var.project_id + network_ip = null + nic_type = var.nic_type + stack_type = null + queue_count = null + access_config = [] + ipv6_access_config = [] + alias_ip_range = [] + } + ] +} + +module "vpc" { + source = "terraform-google-modules/network/google" + version = "~> 10.0" + + network_name = local.network_name + project_id = var.project_id + auto_create_subnetworks = false + subnets = local.template_subnetworks + routing_mode = var.network_routing_mode + mtu = var.mtu + description = var.network_description + shared_vpc_host = var.shared_vpc_host + delete_default_internet_gateway_routes = var.delete_default_internet_gateway_routes + firewall_rules = local.firewall_rules + network_profile = var.network_profile +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf new file mode 100644 index 0000000000..0a21f1d3f2 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf @@ -0,0 +1,59 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "network_name" { + description = "Name of the new VPC network" + value = module.vpc.network_name + depends_on = [module.vpc] +} + +output "network_id" { + description = "ID of the new VPC network" + value = module.vpc.network_id + depends_on = [module.vpc] +} + +output "network_self_link" { + description = "Self link of the new VPC network" + value = module.vpc.network_self_link + depends_on = [module.vpc] +} + +output "subnetworks" { + description = "Full list of subnetwork objects belonging to the new VPC network" + value = module.vpc.subnets + depends_on = [module.vpc] +} + +output "subnetwork_interfaces" { + description = "Full list of subnetwork objects belonging to the new VPC network (compatible with vm-instance and Slurm modules)" + value = local.output_subnets + depends_on = [module.vpc] +} + +# The output subnetwork_interfaces is compatible with vm-instance module but not with gke-node-pool +# See https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/99493df21cecf6a092c45298bf7a45e0343cf622/modules/compute/vm-instance/variables.tf#L220 +# So, we need a separate output that makes the network and subnetwork names available +output "subnetwork_interfaces_gke" { + description = "Full list of subnetwork objects belonging to the new VPC network (compatible with gke-node-pool)" + value = local.output_subnets_gke + depends_on = [module.vpc] +} + +output "subnetwork_name_prefix" { + description = "Prefix of the RDMA subnetwork names" + value = var.subnetworks_template.name_prefix +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf new file mode 100644 index 0000000000..a30fb50e7d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf @@ -0,0 +1,164 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "network_name" { + description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-net\")" + type = string + default = null +} + +variable "region" { + description = "The default region for Cloud resources" + type = string +} + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "mtu" { + type = number + description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." + default = 8896 +} + +variable "subnetworks_template" { + description = <<-EOT + Specifications for the subnetworks that will be created within this VPC. + + count (number, required, number of subnets to create, default is 8) + name_prefix (string, required, subnet name prefix, default is deployment name) + ip_range (string, required, range of IPs for all subnets to share (CIDR format), default is 192.168.0.0/16) + region (string, optional, region to deploy subnets to, defaults to vars.region) + EOT + nullable = false + type = object({ + count = number + name_prefix = string + ip_range = string + region = optional(string) + }) + default = { + count = 8 + name_prefix = null + ip_range = "192.168.0.0/16" + region = null + } + + validation { + condition = var.subnetworks_template.count > 0 + error_message = "Number of subnetworks must be greater than 0" + } + + validation { + condition = can(cidrhost(var.subnetworks_template.ip_range, 0)) + error_message = "IP address range must be in CIDR format." + } +} + +variable "network_routing_mode" { + type = string + default = "REGIONAL" + description = "The network routing mode (default \"REGIONAL\")" + + validation { + condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) + error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." + } +} + +variable "network_description" { + type = string + description = "An optional description of this resource (changes will trigger resource destroy/create)" + default = "" +} + +variable "shared_vpc_host" { + type = bool + description = "Makes this project a Shared VPC host if 'true' (default 'false')" + default = false +} + +variable "delete_default_internet_gateway_routes" { + type = bool + description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" + default = false +} + +variable "enable_internal_traffic" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: enable_internal_traffic can not be specified for gpu-rdma-vpc." + type = bool + default = null + validation { + condition = var.enable_internal_traffic == null + error_message = "DEPRECATED: enable_internal_traffic can not be specified for gpu-rdma-vpc." + } +} + +variable "firewall_rules" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: firewall_rules can not be specified for gpu-rdma-vpc." + type = any + default = null + validation { + condition = var.firewall_rules == null + error_message = "DEPRECATED: firewall_rules can not be specified for gpu-rdma-vpc." + } +} + +variable "firewall_log_config" { # tflint-ignore: terraform_unused_declarations + description = "DEPRECATED: firewall_log_config can not be specified for gpu-rdma-vpc." + type = string + default = null + validation { + condition = var.firewall_log_config == null + error_message = "DEPRECATED: firewall_log_config can not be specified for gpu-rdma-vpc." + } +} + +variable "network_profile" { + description = <<-EOT + A full or partial URL of the network profile to apply to this network. + This field can be set only at resource creation time. For example, the + following are valid URLs: + - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} + - projects/{projectId}/global/networkProfiles/{network_profile_name}} + EOT + type = string + nullable = false + + validation { + condition = can(coalesce(var.network_profile)) + error_message = "var.network_profile must be specified and not an empty string" + } +} + +variable "nic_type" { + description = "NIC type for use in modules that use the output" + type = string + nullable = true + default = "MRDMA" + + validation { + condition = contains(["MRDMA"], var.nic_type) + error_message = "The nic_type must be \"MRDMA\"." + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf new file mode 100644 index 0000000000..71b7106734 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 0.15.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/multivpc/README.md b/vdi-test-scott/primary/modules/embedded/modules/network/multivpc/README.md new file mode 100644 index 0000000000..973e6b32c9 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/multivpc/README.md @@ -0,0 +1,136 @@ +## Description + +This module accomplishes the following: + +* Creates 2 to 8 [VPC networks][vpc] + * Each VPC contains exactly 1 subnetwork + * Each subnetwork contains distinct IP address ranges +* Outputs the `additional_networks` parameter, which is compatible with Slurm + modules + +There are 4 variables that differentiate this module from the standard VPC +module. + +1. `network_prefix`: The name prefix of the VPCs to be created. All + networks and subnetworks will start with this and end with a unique number. +1. `network_count`: The number of VPCs to be created. +1. `global_ip_address_range`: [CIDR-formatted IP range][cidr] +1. `network_cidr_suffix`: The CIDR suffix that defines the address + space that the individual VPCs will cover. + +> [!WARNING] +> The `network_cidr_suffix` should be always be larger than the CIDR suffix on +> `global_ip_address_range`. The difference between these two suffixes should +> be large enough to accommodate the number of VPCs that are being deployed +> (e.g. CIDR suffix bit difference <= `ceil(log2(network_count)))`). + +> [!NOTE] +> For deployments that need multiple VPCs that do not meet this use-case, users +> should deploy multiple individual VPC modules. + +[vpc]: ../vpc/README.md +[cidr]: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation + +### Example + +This snippet uses the multivpc module to create 8 new VPC networks named +`multivpc-net-#` where # ranges from 0 to 7. Additionally, it creates 1 +subnetwork in each VPC. + +```yaml + - id: network + source: modules/network/vpc + + - id: multinetwork + source: modules/network/multivpc + settings: + network_name_prefix: multivpc-net + network_count: 8 + global_ip_address_range: 172.16.0.0/12 + subnetwork_cidr_suffix: 16 + + - id: a3_nodeset + source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset + use: [network, multinetwork] + settings: + machine_type: a3-highgpu-8g + ... +``` + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | + +## Providers + +| Name | Version | +|------|---------| +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [vpcs](#module\_vpcs) | ../vpc | n/a | + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.global_ip_cidr_suffix](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allowed\_ssh\_ip\_ranges](#input\_allowed\_ssh\_ip\_ranges) | A list of CIDR IP ranges from which to allow ssh access | `list(string)` | `[]` | no | +| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [enable\_iap\_rdp\_ingress](#input\_enable\_iap\_rdp\_ingress) | Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels | `bool` | `false` | no | +| [enable\_iap\_ssh\_ingress](#input\_enable\_iap\_ssh\_ingress) | Enable a firewall rule to allow SSH access using IAP tunnels | `bool` | `true` | no | +| [enable\_iap\_winrm\_ingress](#input\_enable\_iap\_winrm\_ingress) | Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels | `bool` | `false` | no | +| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network | `bool` | `true` | no | +| [extra\_iap\_ports](#input\_extra\_iap\_ports) | A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable\_iap variables for standard ports) | `list(string)` | `[]` | no | +| [firewall\_rules](#input\_firewall\_rules) | List of firewall rules | `any` | `[]` | no | +| [global\_ip\_address\_range](#input\_global\_ip\_address\_range) | IP address range (CIDR) that will span entire set of VPC networks | `string` | `"172.16.0.0/12"` | no | +| [ips\_per\_nat](#input\_ips\_per\_nat) | The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT) | `number` | `2` | no | +| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | +| [network\_count](#input\_network\_count) | The number of vpc nettworks to create | `number` | `4` | no | +| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | +| [network\_interface\_defaults](#input\_network\_interface\_defaults) | The template of the network settings to be used on all vpcs. | 
object({
    network            = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string, "GVNIC")
    stack_type         = optional(string, "IPV4_ONLY")
    queue_count        = optional(string)
    access_config = optional(list(object({
      nat_ip                 = string
      network_tier           = string
      public_ptr_domain_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier           = string
      public_ptr_domain_name = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
  })
| 
{
  "access_config": [],
  "alias_ip_range": [],
  "ipv6_access_config": [],
  "network": null,
  "network_ip": "",
  "nic_type": "GVNIC",
  "queue_count": null,
  "stack_type": "IPV4_ONLY",
  "subnetwork": null,
  "subnetwork_project": null
}
| no | +| [network\_name\_prefix](#input\_network\_name\_prefix) | The base name of the vpcs and their subnets, will be appended with a sequence number | `string` | `""` | no | +| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}}
When using a Mellanox network profile (contains 'roce'), if firewall\_rules is specified or enable\_internal\_traffic is true, an error will be thrown | `string` | `null` | no | +| [network\_routing\_mode](#input\_network\_routing\_mode) | The network dynamic routing mode | `string` | `"REGIONAL"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | +| [subnetwork\_cidr\_suffix](#input\_subnetwork\_cidr\_suffix) | The size, in CIDR suffix notation, for each network (e.g. 24 for 172.16.0.0/24); changing this will destroy every network. | `number` | `16` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [additional\_networks](#output\_additional\_networks) | Network interfaces for each subnetwork created by this module | +| [network\_ids](#output\_network\_ids) | IDs of the new VPC network | +| [network\_names](#output\_network\_names) | Names of the new VPC networks | +| [network\_self\_links](#output\_network\_self\_links) | Self link of the new VPC network | +| [subnetwork\_addresses](#output\_subnetwork\_addresses) | IP address range of the primary subnetwork | +| [subnetwork\_names](#output\_subnetwork\_names) | Names of the subnetwork created in each network | +| [subnetwork\_self\_links](#output\_subnetwork\_self\_links) | Self link of the primary subnetwork | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/multivpc/main.tf b/vdi-test-scott/primary/modules/embedded/modules/network/multivpc/main.tf new file mode 100644 index 0000000000..ad06e793c1 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/multivpc/main.tf @@ -0,0 +1,78 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + # this input variable is validated to be in CIDR format + network_name = coalesce(replace(var.network_name_prefix, "_", "-"), replace(var.deployment_name, "_", "-")) + global_ip_cidr_prefix = split("/", var.global_ip_address_range)[0] + global_ip_cidr_suffix = split("/", var.global_ip_address_range)[1] + global_ip_cidr_valid = "${local.global_ip_cidr_prefix}/${terraform_data.global_ip_cidr_suffix.output}" + subnetwork_new_bits = var.subnetwork_cidr_suffix - local.global_ip_cidr_suffix + maximum_subnetworks = pow(2, local.subnetwork_new_bits) + additional_networks = [ + for vpc in module.vpcs : + merge(var.network_interface_defaults, { + network = vpc.network_name + subnetwork = vpc.subnetwork_name + subnetwork_project = var.project_id + }) + ] +} + +resource "terraform_data" "global_ip_cidr_suffix" { + input = local.global_ip_cidr_suffix + lifecycle { + precondition { + condition = local.maximum_subnetworks >= var.network_count + error_message = < 1 + error_message = "The minimum VPCs able to be created by this module is 2. Use the standard Toolkit module at modules/network/vpc for count = 1" + } + validation { + condition = var.network_count <= 8 + error_message = "The maximum VPCs able to be created by this module is 8" + } +} + +variable "global_ip_address_range" { + description = "IP address range (CIDR) that will span entire set of VPC networks" + type = string + default = "172.16.0.0/12" + + validation { + condition = can(cidrhost(var.global_ip_address_range, 0)) + error_message = "var.global_ip_address_range must be an IPv4 CIDR range (e.g. \"172.16.0.0/12\")." + } +} + +variable "subnetwork_cidr_suffix" { + description = "The size, in CIDR suffix notation, for each network (e.g. 24 for 172.16.0.0/24); changing this will destroy every network." + type = number + default = 16 +} + +variable "mtu" { + type = number + description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." + default = 8896 +} + +variable "network_routing_mode" { + type = string + default = "REGIONAL" + description = "The network dynamic routing mode" + + validation { + condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) + error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." + } +} + +variable "network_description" { + type = string + description = "An optional description of this resource (changes will trigger resource destroy/create)" + default = "" +} + +variable "ips_per_nat" { + type = number + description = "The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT)" + default = 2 +} + +variable "delete_default_internet_gateway_routes" { + type = bool + description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" + default = false +} + +variable "enable_iap_ssh_ingress" { + type = bool + description = "Enable a firewall rule to allow SSH access using IAP tunnels" + default = true +} + +variable "enable_iap_rdp_ingress" { + type = bool + description = "Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels" + default = false +} + +variable "enable_iap_winrm_ingress" { + type = bool + description = "Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels" + default = false +} + +variable "enable_internal_traffic" { + type = bool + description = "Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network" + default = true +} + +variable "extra_iap_ports" { + type = list(string) + description = "A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable_iap variables for standard ports)" + default = [] +} + +variable "allowed_ssh_ip_ranges" { + type = list(string) + description = "A list of CIDR IP ranges from which to allow ssh access" + default = [] + + validation { + condition = alltrue([for r in var.allowed_ssh_ip_ranges : can(cidrhost(r, 32))]) + error_message = "Each element of var.allowed_ssh_ip_ranges must be a valid CIDR-formatted IPv4 range." + } +} + +variable "firewall_rules" { + type = any + description = "List of firewall rules" + default = [] +} + +variable "network_interface_defaults" { + type = object({ + network = optional(string) + subnetwork = optional(string) + subnetwork_project = optional(string) + network_ip = optional(string, "") + nic_type = optional(string, "GVNIC") + stack_type = optional(string, "IPV4_ONLY") + queue_count = optional(string) + access_config = optional(list(object({ + nat_ip = string + network_tier = string + public_ptr_domain_name = string + })), []) + ipv6_access_config = optional(list(object({ + network_tier = string + public_ptr_domain_name = string + })), []) + alias_ip_range = optional(list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })), []) + }) + description = "The template of the network settings to be used on all vpcs." + default = { + network = null + subnetwork = null + subnetwork_project = null + network_ip = "" + nic_type = "GVNIC" + stack_type = "IPV4_ONLY" + queue_count = null + access_config = [] + ipv6_access_config = [] + alias_ip_range = [] + } +} + +variable "network_profile" { + type = string + description = <<-EOT + A full or partial URL of the network profile to apply to this network. + This field can be set only at resource creation time. For example, the + following are valid URLs: + - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} + - projects/{projectId}/global/networkProfiles/{network_profile_name}} + When using a Mellanox network profile (contains 'roce'), if firewall_rules is specified or enable_internal_traffic is true, an error will be thrown + EOT + default = null +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/multivpc/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/network/multivpc/versions.tf new file mode 100644 index 0000000000..e75a67f7b6 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/multivpc/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 1.4.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/README.md b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/README.md new file mode 100644 index 0000000000..4d63b17091 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/README.md @@ -0,0 +1,94 @@ +## Description + +This module discovers a subnetwork that already exists in Google Cloud and +outputs subnetwork attributes that uniquely identify it for use by other modules. + +For example, the blueprint below discovers the referred to subnetwork. +With the `use` keyword, the [vm-instance] module accepts the `subnetwork_self_link` +input variables that uniquely identify the subnetwork in which the VM will be created. + +[vpc]: ../vpc/README.md +[vm-instance]: ../../compute/vm-instance/README.md + +> **_NOTE:_** Additional IAM work is needed for this to work correctly. + +### Example + +```yaml +- id: network + source: modules/network/pre-existing-subnetwork + settings: + subnetwork_self_link: https://www.googleapis.com/compute/v1/projects/name-of-host-project/regions/REGION/subnetworks/SUBNETNAME + +- id: example_vm + source: modules/compute/vm-instance + use: + - network + settings: + name_prefix: example + machine_type: c2-standard-4 +``` + +As described in documentation: +[https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork] + +If subnetwork_self_link is provided then name,region,project is ignored. + +## License + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_subnetwork.primary_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [project](#input\_project) | Name of the project that owns the subnetwork | `string` | `null` | no | +| [region](#input\_region) | Region in which to search for primary subnetwork | `string` | `null` | no | +| [subnetwork\_name](#input\_subnetwork\_name) | Name of the pre-existing VPC subnetwork | `string` | `null` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Self-link of the subnet in the VPC | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [subnetwork](#output\_subnetwork) | Full subnetwork object in the primary region | +| [subnetwork\_address](#output\_subnetwork\_address) | Subnetwork IP range in the primary region | +| [subnetwork\_name](#output\_subnetwork\_name) | Name of the subnetwork in the primary region | +| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Subnetwork self-link in the primary region | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/main.tf b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/main.tf new file mode 100644 index 0000000000..9fb206f969 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/main.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +data "google_compute_subnetwork" "primary_subnetwork" { + name = var.subnetwork_name + region = var.region + project = var.project + self_link = var.subnetwork_self_link + + lifecycle { + postcondition { + condition = self.self_link != null + error_message = "The subnetwork: ${coalesce(var.subnetwork_name, var.subnetwork_self_link)} could not be found." + } + } +} + +# Module-level check for Private Google Access on the subnetwork +check "private_google_access_enabled_subnetwork" { + assert { + condition = data.google_compute_subnetwork.primary_subnetwork.private_ip_google_access + error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.primary_subnetwork.name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml new file mode 100644 index 0000000000..6a6f1e5757 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com +ghpc: + has_to_be_used: true diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf new file mode 100644 index 0000000000..868708dc6b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "subnetwork" { + description = "Full subnetwork object in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork +} + +output "subnetwork_name" { + description = "Name of the subnetwork in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.name +} + +output "subnetwork_self_link" { + description = "Subnetwork self-link in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.self_link +} + +output "subnetwork_address" { + description = "Subnetwork IP range in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.ip_cidr_range +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf new file mode 100644 index 0000000000..d5191843e8 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "subnetwork_self_link" { + description = "Self-link of the subnet in the VPC" + type = string + default = null +} + +variable "project" { + description = "Name of the project that owns the subnetwork" + type = string + default = null +} + +variable "subnetwork_name" { + description = "Name of the pre-existing VPC subnetwork" + type = string + default = null +} + +variable "region" { + description = "Region in which to search for primary subnetwork" + type = string + default = null +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf new file mode 100644 index 0000000000..e90ae5991d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:pre-existing-subnetwork/v1.57.0" + } + + required_version = ">= 1.5" +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/README.md b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/README.md new file mode 100644 index 0000000000..38a1840c2d --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/README.md @@ -0,0 +1,110 @@ +## Description + +This module discovers a VPC network that already exists in Google Cloud and +outputs network attributes that uniquely identify it for use by other modules. +The module outputs are aligned with the [vpc module][vpc] so that it can be used +as a drop-in substitute when a VPC already exists. + +For example, the blueprint below discovers the "default" global network and the +"default" regional subnetwork in us-central1. With the `use` keyword, the +[vm-instance] module accepts the `network_self_link` and `subnetwork_self_link` +input variables that uniquely identify the network and subnetwork in which the +VM will be created. + +[vpc]: ../vpc/README.md +[vm-instance]: ../../compute/vm-instance/README.md + +### Example + +```yaml +- id: network1 + source: modules/network/pre-existing-vpc + settings: + project_id: $(vars.project_id) + region: us-central1 + +- id: example_vm + source: modules/compute/vm-instance + use: + - network1 + settings: + name_prefix: example + machine_type: c2-standard-4 +``` + +> **_NOTE:_** The `project_id` and `region` settings would be inferred from the +> deployment variables of the same name, but they are included here for clarity. + +### Use shared-vpc + +If a network is created in different project, this module can be used to +reference the network. To use a network from a different project first make sure +you have a [cloud nat][cloudnat] and [IAP][iap] forwarding. For more details, +refer [shared-vpc][shared-vpc-doc] + +[cloudnat]: https://cloud.google.com/nat/docs/overview +[iap]: https://cloud.google.com/iap/docs/using-tcp-forwarding +[shared-vpc-doc]: ../../../examples/README.md#hpc-slurm-sharedvpcyaml-community-badge-experimental-badge + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_compute_network.vpc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network) | data source | +| [google_compute_subnetwork.primary_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [network\_name](#input\_network\_name) | Name of the existing VPC network | `string` | `"default"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | Region in which to search for primary subnetwork | `string` | n/a | yes | +| [subnetwork\_name](#input\_subnetwork\_name) | Name of the pre-existing VPC subnetwork; defaults to var.network\_name if set to null. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [network\_id](#output\_network\_id) | ID of the existing VPC network | +| [network\_name](#output\_network\_name) | Name of the existing VPC network | +| [network\_self\_link](#output\_network\_self\_link) | Self link of the existing VPC network | +| [subnetwork](#output\_subnetwork) | Full subnetwork object in the primary region | +| [subnetwork\_address](#output\_subnetwork\_address) | Subnetwork IP range in the primary region | +| [subnetwork\_name](#output\_subnetwork\_name) | Name of the subnetwork in the primary region | +| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Subnetwork self-link in the primary region | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/main.tf b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/main.tf new file mode 100644 index 0000000000..ed332bab72 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/main.tf @@ -0,0 +1,53 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +data "google_compute_network" "vpc" { + name = var.network_name + project = var.project_id + + lifecycle { + postcondition { + condition = self.self_link != null + error_message = "The network: ${var.network_name} could not be found in project: ${var.project_id}." + } + } +} + +locals { + subnetwork_name = var.subnetwork_name != null ? var.subnetwork_name : var.network_name +} + +data "google_compute_subnetwork" "primary_subnetwork" { + name = local.subnetwork_name + region = var.region + project = var.project_id + + lifecycle { + postcondition { + condition = self.self_link != null + error_message = "The subnetwork: ${local.subnetwork_name} could not be found in project: ${var.project_id} and region: ${var.region}." + } + } +} + +# Module-level check for Private Google Access on the subnetwork +check "private_google_access_enabled_subnetwork" { + assert { + condition = data.google_compute_subnetwork.primary_subnetwork.private_ip_google_access + error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.primary_subnetwork.name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/outputs.tf new file mode 100644 index 0000000000..00861af5ca --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/outputs.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "network_name" { + description = "Name of the existing VPC network" + value = data.google_compute_network.vpc.name +} + +output "network_id" { + description = "ID of the existing VPC network" + value = data.google_compute_network.vpc.id +} + +output "network_self_link" { + description = "Self link of the existing VPC network" + value = data.google_compute_network.vpc.self_link +} + +output "subnetwork" { + description = "Full subnetwork object in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork +} + +output "subnetwork_name" { + description = "Name of the subnetwork in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.name +} + +output "subnetwork_self_link" { + description = "Subnetwork self-link in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.self_link +} + +output "subnetwork_address" { + description = "Subnetwork IP range in the primary region" + value = data.google_compute_subnetwork.primary_subnetwork.ip_cidr_range +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/variables.tf new file mode 100644 index 0000000000..291a81604a --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/variables.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "network_name" { + description = "Name of the existing VPC network" + type = string + default = "default" +} + +variable "subnetwork_name" { + description = "Name of the pre-existing VPC subnetwork; defaults to var.network_name if set to null." + type = string + default = null +} + +variable "region" { + description = "Region in which to search for primary subnetwork" + type = string +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/versions.tf new file mode 100644 index 0000000000..d798ca3f8c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:pre-existing-vpc/v1.57.0" + } + + required_version = ">= 1.5" +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/vpc/README.md b/vdi-test-scott/primary/modules/embedded/modules/network/vpc/README.md new file mode 100644 index 0000000000..7521537a9f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/vpc/README.md @@ -0,0 +1,237 @@ +## Description + +This module creates a new [VPC network][vpc] with 1 or more subnetworks and +a [Cloud Router][router] for every region with a subnetwork. By default, it will +create: + +* A [Cloud NAT][nat] to enable outbound access to the public internet for VMs + without public IP addresses; VMs with public IP addresses bypass the NAT to + directly access the public internet +* A firewall rule that enables inbound SSH access from [Identity-Aware + Proxy][iap] +* A firewall rule that enables all traffic internal to the network + +This behavior is optional and can be configured as [described below](#inputs). +This module is based on networking support in the [Cloud Foundation +Toolkit][cft]. We recommend following the [documentation for the network +module][cft-network] and [submodules][cft-network-submodules] for more details. +In particular, the detailed structure of input variables can be found for: + +* [var.firewall\_rules](https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/firewall-rules#inputs) +* [var.secondary\_ranges](https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/subnets#inputs) + +[vpc]: https://cloud.google.com/vpc +[router]: https://github.com/terraform-google-modules/terraform-google-cloud-router +[nat]: https://github.com/terraform-google-modules/terraform-google-cloud-nat +[iap]: https://cloud.google.com/iap +[cft]: https://cloud.google.com/foundation-toolkit +[cft-network]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0 +[cft-network-submodules]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules + +Additionally, [Google Private Access][gpa] is enabled by default on all +subnetworks unless it is explicitly disabled. This setting ensures that all VMs +can use Google services such as [Cloud Storage][gcs] even if they do not have +public IP addresses or Cloud NAT is disabled. + +[gpa]: https://cloud.google.com/vpc/docs/private-google-access +[gcs]: https://cloud.google.com/storage + +### Example + +This creates a new VPC network named `cluster-net`. + +```yaml + - id: network1 + source: modules/network/vpc + settings: + network_name: cluster-net +``` + +### Deprecation warning + +The variables listed below have been deprecated and will be removed in a future +release. Until they are removed,You may continue to use them in Toolkit +blueprints with the same functionality as documented in the [Toolkit 1.0 +release][vpc1.0]. + +* Deprecated variables + * `var.primary_subnetwork` + * `var.additional_subnetworks` + * `var.subnetwork_size` + +[vpc1.0]: https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/v1.0.0/modules/network/vpc/README.md + +The following variables have been added to support explicit IP ranges for +subnetworks while retaining existing functionality. We advise adopting them even +if not using explicit IP ranges . The Toolkit ***does not support*** mixing +deprecated variables with the new replacements. The new functionality is +described in [more detail below](#subnetworks). + +* New variables to adopt + * `var.subnetworks` + * A value for this can be generated by merging `var.primary_subnetwork` and + `var.additional_subnetworks` into a single list + * `var.default_primary_subnetwork_size` + * This variable has been renamed for clarity; its value can be directly + copied from an explicit setting for `var.subnetwork_size`; if your blueprint + does not have an explicit setting, the default values are the same + +### Subnetworks + +This module will always provision at least 1 "primary" subnetwork in which most +resources are expected to be provisioned. This primary subnetwork is determined +by + +1. The first element of [var.subnetworks](#input_subnetworks) if it is not the + empty list +2. A default subnetwork automatically calculated from + * [var.subnetwork_name](#input_subnetwork_name) + * [var.region](#input_region) + * [var.network_address_range](#input_network_address_range) + * [var.default_primary_subnetwork_size](#input_default_primary_subnetwork_size) + +If `var.subnetworks` is provided then the primary subnetwork name is taken +explicitly from it and `var.subnetwork_name` is ignored. + +`var.subnetworks` behaves identically to the [Cloud Foundation Toolkit subnets +module][cftsubnets] with the lone exception that one can provide ***one*** of +the following settings for each subnetwork: + +* `new_bits` +* `subnet_ip` + +If each subnetwork defines `subnet_ip` then these are taken to be their explicit +CIDR IP ranges. If each subnetwork defines `new_bits`, then these are taken to +be the size of the CIDR subnetwork (in bits). IP ranges for each subnetwork are +calculated using `var.network_address_range` as the base IP, producing the most +compact set of subnetworks possible. + +> **_NOTE:_** we do not presently support the modification of individual subnetworks +> when using this module to provision more than 1 subnetwork using automatically +> calculated IP ranges based upon `new_bits`. Doing so will cause IP ranges to be +> recalculated for each subnetwork. We advise appending new subnetworks to the end +> of `var.subnetworks`. + +[cftsubnets]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/subnets + +### SSH Access + +By default a firewall rule is created to allow inbound SSH access from +[Identity-Aware Proxy][iap]. A user must have the `IAP-Secured Tunnel User` +(`roles/iap.tunnelResourceAccessor`) IAM role to be able to SSH over IAP. + +To allow regular SSH access from a known IP address you can add the following +`firewall_rules` setting to the `vpc` module: + +```yaml + - id: network1 + source: modules/network/vpc + settings: + firewall_rules: + - name: ssh-my-machine + direction: INGRESS + ranges: [/32] + allow: + - protocol: tcp + ports: [22] +``` + +> **Note**: You must populate the above example with the source IP address from +> which you plan to SSH from. You can use a service like +> [whatismyip.com](https://whatismyip.com) to determine your IP address. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15.0 | + +## Providers + +| Name | Version | +|------|---------| +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [cloud\_router](#module\_cloud\_router) | terraform-google-modules/cloud-router/google | ~> 6.0 | +| [nat\_ip\_addresses](#module\_nat\_ip\_addresses) | terraform-google-modules/address/google | ~> 4.1 | +| [vpc](#module\_vpc) | terraform-google-modules/network/google | ~> 10.0 | + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.cloud_nat_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.network_profile_firewall_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.secondary_ranges_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_subnetworks](#input\_additional\_subnetworks) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `list(map(string))` | `null` | no | +| [allowed\_ssh\_ip\_ranges](#input\_allowed\_ssh\_ip\_ranges) | A list of CIDR IP ranges from which to allow ssh access | `list(string)` | `[]` | no | +| [default\_primary\_subnetwork\_size](#input\_default\_primary\_subnetwork\_size) | The size, in CIDR bits, of the default primary subnetwork unless explicitly defined in var.subnetworks | `number` | `15` | no | +| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [enable\_cloud\_nat](#input\_enable\_cloud\_nat) | Enable the creation of Cloud NATs. | `bool` | `true` | no | +| [enable\_cloud\_router](#input\_enable\_cloud\_router) | Enable the creation of a Cloud Router for your VPC. For more information on Cloud Routers see https://cloud.google.com/network-connectivity/docs/router/concepts/overview | `bool` | `true` | no | +| [enable\_iap\_rdp\_ingress](#input\_enable\_iap\_rdp\_ingress) | Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels | `bool` | `false` | no | +| [enable\_iap\_ssh\_ingress](#input\_enable\_iap\_ssh\_ingress) | Enable a firewall rule to allow SSH access using IAP tunnels | `bool` | `true` | no | +| [enable\_iap\_winrm\_ingress](#input\_enable\_iap\_winrm\_ingress) | Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels | `bool` | `false` | no | +| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network | `bool` | `true` | no | +| [extra\_iap\_ports](#input\_extra\_iap\_ports) | A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable\_iap variables for standard ports) | `list(string)` | `[]` | no | +| [firewall\_log\_config](#input\_firewall\_log\_config) | Firewall log configuration for Toolkit firewall rules (var.enable\_iap\_ssh\_ingress and others) | `string` | `"DISABLE_LOGGING"` | no | +| [firewall\_rules](#input\_firewall\_rules) | List of firewall rules | `any` | `[]` | no | +| [ips\_per\_nat](#input\_ips\_per\_nat) | The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT) | `number` | `2` | no | +| [labels](#input\_labels) | Labels to add to network resources that support labels. Key-value pairs of strings. | `map(string)` | `{}` | no | +| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | +| [network\_address\_range](#input\_network\_address\_range) | IP address range (CIDR) for global network | `string` | `"10.0.0.0/9"` | no | +| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | +| [network\_name](#input\_network\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-net") | `string` | `null` | no | +| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}}
When using a Mellanox network profile (contains 'roce'), if firewall\_rules is specified or enable\_internal\_traffic is true, an error will be thrown | `string` | `null` | no | +| [network\_routing\_mode](#input\_network\_routing\_mode) | The network routing mode (default "GLOBAL") | `string` | `"GLOBAL"` | no | +| [primary\_subnetwork](#input\_primary\_subnetwork) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `map(string)` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | +| [secondary\_ranges](#input\_secondary\_ranges) | "Secondary ranges associated with the subnets.
This will be deprecated in favour of secondary\_ranges\_list at a later date.
Please migrate to using the same." | `map(list(object({ range_name = string, ip_cidr_range = string })))` | `{}` | no | +| [secondary\_ranges\_list](#input\_secondary\_ranges\_list) | "List of secondary ranges associated with the subnetworks.
Each subnetwork must be specified at most once in this list." | 
list(object({
    subnetwork_name = string,
    ranges = list(object({
      range_name    = string,
      ip_cidr_range = string
    }))
  }))
| `[]` | no | +| [shared\_vpc\_host](#input\_shared\_vpc\_host) | Makes this project a Shared VPC host if 'true' (default 'false') | `bool` | `false` | no | +| [subnetwork\_name](#input\_subnetwork\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-primary-subnet") | `string` | `null` | no | +| [subnetwork\_size](#input\_subnetwork\_size) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `number` | `null` | no | +| [subnetworks](#input\_subnetworks) | List of subnetworks to create within the VPC. If left empty, it will be
replaced by a single, default subnetwork constructed from other parameters
(e.g. var.region). In all cases, the first subnetwork in the list is identified
by outputs as a "primary" subnetwork.

subnet\_name (string, required, name of subnet)
subnet\_region (string, required, region of subnet)
subnet\_ip (string, mutually exclusive with new\_bits, CIDR-formatted IP range for subnetwork)
new\_bits (number, mutually exclusive with subnet\_ip, CIDR bits used to calculate subnetwork range)
subnet\_private\_access (bool, optional, Enable Private Access on subnetwork)
subnet\_flow\_logs (map(string), optional, Configure Flow Logs see terraform-google-network module)
description (string, optional, Description of Network)
purpose (string, optional, related to Load Balancing)
role (string, optional, related to Load Balancing) | `list(map(string))` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [nat\_ips](#output\_nat\_ips) | External IPs of the Cloud NAT from which outbound internet traffic will arrive (empty list if no NAT is used) | +| [network\_id](#output\_network\_id) | ID of the new VPC network | +| [network\_name](#output\_network\_name) | Name of the new VPC network | +| [network\_self\_link](#output\_network\_self\_link) | Self link of the new VPC network | +| [subnetwork](#output\_subnetwork) | Primary subnetwork object | +| [subnetwork\_address](#output\_subnetwork\_address) | IP address range of the primary subnetwork | +| [subnetwork\_name](#output\_subnetwork\_name) | Name of the primary subnetwork | +| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Self link of the primary subnetwork | +| [subnetworks](#output\_subnetworks) | Full list of subnetwork objects belonging to the new VPC network | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/vpc/main.tf b/vdi-test-scott/primary/modules/embedded/modules/network/vpc/main.tf new file mode 100644 index 0000000000..fecfc73df3 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/vpc/main.tf @@ -0,0 +1,256 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +resource "terraform_data" "secondary_ranges_validation" { + lifecycle { + precondition { + condition = !(length(var.secondary_ranges) > 0 && length(var.secondary_ranges_list) > 0) + error_message = "Only one of var.secondary_ranges or var.secondary_ranges_list should be specified" + } + } +} + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "vpc", ghpc_role = "network" }) +} + +locals { + autoname = replace(var.deployment_name, "_", "-") + network_name = var.network_name == null ? "${local.autoname}-net" : var.network_name + subnetwork_name = var.subnetwork_name == null ? "${local.autoname}-primary-subnet" : var.subnetwork_name + + # define a default subnetwork for cases in which no explicit subnetworks are + # defined in var.subnetworks + default_primary_subnetwork_cidr_block = cidrsubnet(var.network_address_range, var.default_primary_subnetwork_size, 0) + default_primary_subnetwork = { + subnet_name = local.subnetwork_name + subnet_ip = local.default_primary_subnetwork_cidr_block + subnet_region = var.region + subnet_private_access = true + subnet_flow_logs = false + description = "primary subnetwork in ${local.network_name}" + purpose = null + role = null + } + + # Identify user-supplied primary subnetwork + # (1) explicit var.subnetworks[0] + # (2) implicit local default subnetwork + input_primary_subnetwork = coalesce(try(var.subnetworks[0], null), local.default_primary_subnetwork) + + # Identify user-supplied additional subnetworks + # (1) explicit var.subnetworks[1:end] + # (2) empty list + input_additional_subnetworks = try(slice(var.subnetworks, 1, length(var.subnetworks)), []) + + # at this point we have constructed a list of subnetworks but need to extract + # user-provided CIDR blocks or calculate them from user-provided new_bits + # after we complete deprecation, local.all_subnetworks can be replaced with + # var.subnetworks (or local.default_primary_subnetwork if that is null) + input_subnetworks = concat([local.input_primary_subnetwork], local.input_additional_subnetworks) + subnetworks_cidr_blocks = try( + local.input_subnetworks[*]["subnet_ip"], + cidrsubnets(var.network_address_range, local.input_subnetworks[*]["new_bits"]...) + ) + + # merge in the CIDR blocks (even when already there) and remove new_bits + subnetworks = [for i, subnet in local.input_subnetworks : + merge({ for k, v in subnet : k => v if k != "new_bits" }, { "subnet_ip" = local.subnetworks_cidr_blocks[i] }) + ] + + # gather the unique regions for purposes of creating Router/NAT + cloud_router_regions = var.enable_cloud_router ? distinct([for subnet in local.subnetworks : subnet.subnet_region]) : [] + cloud_nat_regions = var.enable_cloud_nat ? local.cloud_router_regions : [] + + # this comprehension should have 1 and only 1 match + output_primary_subnetwork = one([for k, v in module.vpc.subnets : v if k == "${local.subnetworks[0].subnet_region}/${local.subnetworks[0].subnet_name}"]) + output_primary_subnetwork_name = local.output_primary_subnetwork.name + output_primary_subnetwork_self_link = local.output_primary_subnetwork.self_link + output_primary_subnetwork_ip_cidr_range = local.output_primary_subnetwork.ip_cidr_range + + iap_ports = distinct(concat(compact([ + var.enable_iap_rdp_ingress ? "3389" : "", + var.enable_iap_ssh_ingress ? "22" : "", + var.enable_iap_winrm_ingress ? "5986" : "", + ]), var.extra_iap_ports)) + + firewall_log_api_values = { + "DISABLE_LOGGING" = null + "INCLUDE_ALL_METADATA" = { metadata = "INCLUDE_ALL_METADATA" }, + "EXCLUDE_ALL_METADATA" = { metadata = "EXCLUDE_ALL_METADATA" }, + } + firewall_log_config = lookup(local.firewall_log_api_values, var.firewall_log_config, null) + + allow_iap_ingress = { + name = "${local.network_name}-fw-allow-iap-ingress" + description = "allow TCP access via Identity-Aware Proxy" + direction = "INGRESS" + priority = null + ranges = ["35.235.240.0/20"] + source_tags = null + source_service_accounts = null + target_tags = null + target_service_accounts = null + allow = [{ + protocol = "tcp" + ports = local.iap_ports + }] + deny = [] + log_config = local.firewall_log_config + } + + allow_ssh_ingress = { + name = "${local.network_name}-fw-allow-ssh-ingress" + description = "allow SSH access" + direction = "INGRESS" + priority = null + ranges = var.allowed_ssh_ip_ranges + source_tags = null + source_service_accounts = null + target_tags = null + target_service_accounts = null + allow = [{ + protocol = "tcp" + ports = ["22"] + }] + deny = [] + log_config = local.firewall_log_config + } + + allow_internal_traffic = { + name = "${local.network_name}-fw-allow-internal-traffic" + priority = null + description = "allow traffic between nodes of this VPC" + direction = "INGRESS" + ranges = [var.network_address_range] + source_tags = null + source_service_accounts = null + target_tags = null + target_service_accounts = null + allow = [{ + protocol = "tcp" + ports = ["0-65535"] + }, { + protocol = "udp" + ports = ["0-65535"] + }, { + protocol = "icmp" + ports = null + }, + ] + deny = [] + log_config = local.firewall_log_config + } + + firewall_rules = concat( + var.firewall_rules, + length(var.allowed_ssh_ip_ranges) > 0 ? [local.allow_ssh_ingress] : [], + var.enable_internal_traffic ? [local.allow_internal_traffic] : [], + length(local.iap_ports) > 0 ? [local.allow_iap_ingress] : [] + ) + + secondary_ranges_map = { + for secondary_range in var.secondary_ranges_list : + secondary_range.subnetwork_name => secondary_range.ranges + } +} + +resource "terraform_data" "network_profile_firewall_validation" { + lifecycle { + precondition { + condition = !(try(strcontains(var.network_profile, "roce"), false) && length(local.firewall_rules) > 0) + error_message = "If var.network_profile contains 'roce', var.firewall_rules must be empty and var.enable_internal_traffic must be false, please see: https://cloud.google.com/vpc/docs/rdma-network-profiles#additional_features_that_dont_apply_to_traffic_from_rdma_nics" + } + } +} + +module "vpc" { + source = "terraform-google-modules/network/google" + version = "~> 10.0" + + depends_on = [terraform_data.network_profile_firewall_validation] + + network_name = local.network_name + project_id = var.project_id + auto_create_subnetworks = false + subnets = local.subnetworks + secondary_ranges = length(local.secondary_ranges_map) > 0 ? local.secondary_ranges_map : var.secondary_ranges + routing_mode = var.network_routing_mode + mtu = var.mtu + description = var.network_description + shared_vpc_host = var.shared_vpc_host + delete_default_internet_gateway_routes = var.delete_default_internet_gateway_routes + firewall_rules = local.firewall_rules + network_profile = var.network_profile +} + +resource "terraform_data" "cloud_nat_validation" { + lifecycle { + precondition { + condition = var.enable_cloud_router == true || var.enable_cloud_nat == false + error_message = <<-EOD + "Cannot have Cloud NAT without a Cloud Router. If you desire Cloud NAT functionality please set `enable_cloud_router` to true." + EOD + } + } +} + +# This use of the module may appear odd when var.ips_per_nat = 0. The module +# will be called for all regions with subnetworks but names will be set to the +# empty list. This is a perfectly valid value (the default!). In this scenario, +# no IP addresses are created and all module outputs are empty lists. +# +# https://github.com/terraform-google-modules/terraform-google-address/blob/v3.1.1/variables.tf#L27 +# https://github.com/terraform-google-modules/terraform-google-address/blob/v3.1.1/outputs.tf +module "nat_ip_addresses" { + source = "terraform-google-modules/address/google" + version = "~> 4.1" + + depends_on = [terraform_data.cloud_nat_validation] + + for_each = toset(local.cloud_nat_regions) + + project_id = var.project_id + region = each.value + # an external, regional (not global) IP address is suited for a regional NAT + address_type = "EXTERNAL" + global = false + labels = local.labels + names = [for idx in range(var.ips_per_nat) : "${local.network_name}-nat-ips-${each.value}-${idx}"] +} + +module "cloud_router" { + source = "terraform-google-modules/cloud-router/google" + version = "~> 6.0" + + depends_on = [terraform_data.cloud_nat_validation] + + for_each = toset(local.cloud_router_regions) + + project = var.project_id + name = "${local.network_name}-router" + region = each.value + network = module.vpc.network_name + # in scenario with no NAT IPs, no NAT is created even if router is created + # https://github.com/terraform-google-modules/terraform-google-cloud-router/blob/v2.0.0/nat.tf#L18-L20 + nats = length(module.nat_ip_addresses[each.value].self_links) == 0 ? [] : [ + { + name : "cloud-nat-${each.value}", + nat_ips : module.nat_ip_addresses[each.value].self_links + }, + ] +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/vpc/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/network/vpc/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/vpc/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/vpc/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/network/vpc/outputs.tf new file mode 100644 index 0000000000..c2ee6bdf6b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/vpc/outputs.tf @@ -0,0 +1,68 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "network_name" { + description = "Name of the new VPC network" + value = module.vpc.network_name + depends_on = [module.vpc, module.cloud_router] +} + +output "network_id" { + description = "ID of the new VPC network" + value = module.vpc.network_id + depends_on = [module.vpc, module.cloud_router] +} + +output "network_self_link" { + description = "Self link of the new VPC network" + value = module.vpc.network_self_link + depends_on = [module.vpc, module.cloud_router] +} + +output "subnetworks" { + description = "Full list of subnetwork objects belonging to the new VPC network" + value = module.vpc.subnets + depends_on = [module.vpc, module.cloud_router] +} + +output "subnetwork" { + description = "Primary subnetwork object" + value = local.output_primary_subnetwork + depends_on = [module.vpc, module.cloud_router] +} + +output "subnetwork_name" { + description = "Name of the primary subnetwork" + value = local.output_primary_subnetwork_name + depends_on = [module.vpc, module.cloud_router] +} + +output "subnetwork_self_link" { + description = "Self link of the primary subnetwork" + value = local.output_primary_subnetwork_self_link + depends_on = [module.vpc, module.cloud_router] +} + +output "subnetwork_address" { + description = "IP address range of the primary subnetwork" + value = local.output_primary_subnetwork_ip_cidr_range + depends_on = [module.vpc, module.cloud_router] +} + +output "nat_ips" { + description = "External IPs of the Cloud NAT from which outbound internet traffic will arrive (empty list if no NAT is used)" + value = flatten([for ipmod in module.nat_ip_addresses : ipmod.addresses]) +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/vpc/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/network/vpc/variables.tf new file mode 100644 index 0000000000..32a9a00e74 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/vpc/variables.tf @@ -0,0 +1,301 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "labels" { + description = "Labels to add to network resources that support labels. Key-value pairs of strings." + type = map(string) + default = {} + nullable = false +} + +variable "network_name" { + description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-net\")" + type = string + default = null +} + +variable "subnetwork_name" { + description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-primary-subnet\")" + type = string + default = null +} + +# tflint-ignore: terraform_unused_declarations +variable "subnetwork_size" { + description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" + type = number + default = null + validation { + condition = var.subnetwork_size == null + error_message = "subnetwork_size is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." + } +} + +variable "default_primary_subnetwork_size" { + description = "The size, in CIDR bits, of the default primary subnetwork unless explicitly defined in var.subnetworks" + type = number + default = 15 +} + +variable "region" { + description = "The default region for Cloud resources" + type = string +} + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "network_address_range" { + description = "IP address range (CIDR) for global network" + type = string + default = "10.0.0.0/9" + + validation { + condition = can(cidrhost(var.network_address_range, 0)) + error_message = "IP address range must be in CIDR format." + } +} + +variable "mtu" { + type = number + description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." + default = 8896 +} + +variable "subnetworks" { + description = <<-EOT + List of subnetworks to create within the VPC. If left empty, it will be + replaced by a single, default subnetwork constructed from other parameters + (e.g. var.region). In all cases, the first subnetwork in the list is identified + by outputs as a "primary" subnetwork. + + subnet_name (string, required, name of subnet) + subnet_region (string, required, region of subnet) + subnet_ip (string, mutually exclusive with new_bits, CIDR-formatted IP range for subnetwork) + new_bits (number, mutually exclusive with subnet_ip, CIDR bits used to calculate subnetwork range) + subnet_private_access (bool, optional, Enable Private Access on subnetwork) + subnet_flow_logs (map(string), optional, Configure Flow Logs see terraform-google-network module) + description (string, optional, Description of Network) + purpose (string, optional, related to Load Balancing) + role (string, optional, related to Load Balancing) + EOT + type = list(map(string)) + default = [] + validation { + condition = alltrue([ + for s in var.subnetworks : can(s["subnet_name"]) + ]) + error_message = "All subnetworks must define \"subnet_name\"." + } + validation { + condition = alltrue([ + for s in var.subnetworks : can(s["subnet_region"]) + ]) + error_message = "All subnetworks must define \"subnet_region\"." + } + validation { + condition = alltrue([ + for s in var.subnetworks : can(s["subnet_ip"]) != can(s["new_bits"]) + ]) + error_message = "All subnetworks must define exactly one of \"subnet_ip\" or \"new_bits\"." + } + validation { + condition = alltrue([for s in var.subnetworks : can(s["subnet_ip"])]) || alltrue([for s in var.subnetworks : can(s["new_bits"])]) + error_message = "All subnetworks must make same choice of \"subnet_ip\" or \"new_bits\"." + } +} + +# tflint-ignore: terraform_unused_declarations +variable "primary_subnetwork" { + description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" + type = map(string) + default = null + validation { + condition = var.primary_subnetwork == null + error_message = "primary_subnetwork is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." + } +} + +# tflint-ignore: terraform_unused_declarations +variable "additional_subnetworks" { + description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" + type = list(map(string)) + default = null + validation { + condition = var.additional_subnetworks == null + error_message = "additional_subnetworks is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." + } +} + +variable "secondary_ranges" { + type = map(list(object({ range_name = string, ip_cidr_range = string }))) + description = <<-EOT + "Secondary ranges associated with the subnets. + This will be deprecated in favour of secondary_ranges_list at a later date. + Please migrate to using the same." + EOT + default = {} +} + +variable "secondary_ranges_list" { + type = list(object({ + subnetwork_name = string, + ranges = list(object({ + range_name = string, + ip_cidr_range = string + })) + })) + description = <<-EOT + "List of secondary ranges associated with the subnetworks. + Each subnetwork must be specified at most once in this list." + EOT + default = [] + validation { + condition = (length(var.secondary_ranges_list[*].subnetwork_name) == + length(distinct(var.secondary_ranges_list[*].subnetwork_name))) + error_message = "Each subnetwork should be specified at most once in this list. Remove any duplicates." + } +} + +variable "network_routing_mode" { + type = string + default = "GLOBAL" + description = "The network routing mode (default \"GLOBAL\")" + + validation { + condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) + error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." + } +} + +variable "network_description" { + type = string + description = "An optional description of this resource (changes will trigger resource destroy/create)" + default = "" +} + +variable "ips_per_nat" { + type = number + description = "The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT)" + default = 2 +} + +variable "shared_vpc_host" { + type = bool + description = "Makes this project a Shared VPC host if 'true' (default 'false')" + default = false +} + +variable "delete_default_internet_gateway_routes" { + type = bool + description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" + default = false +} + +variable "enable_iap_ssh_ingress" { + type = bool + description = "Enable a firewall rule to allow SSH access using IAP tunnels" + default = true +} + +variable "enable_iap_rdp_ingress" { + type = bool + description = "Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels" + default = false +} + +variable "enable_iap_winrm_ingress" { + type = bool + description = "Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels" + default = false +} + +variable "enable_internal_traffic" { + type = bool + description = "Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network" + default = true +} + +variable "enable_cloud_router" { + type = bool + description = "Enable the creation of a Cloud Router for your VPC. For more information on Cloud Routers see https://cloud.google.com/network-connectivity/docs/router/concepts/overview" + default = true +} + +variable "enable_cloud_nat" { + type = bool + description = "Enable the creation of Cloud NATs." + default = true +} + +variable "extra_iap_ports" { + type = list(string) + description = "A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable_iap variables for standard ports)" + default = [] +} + +variable "allowed_ssh_ip_ranges" { + type = list(string) + description = "A list of CIDR IP ranges from which to allow ssh access" + default = [] + + validation { + condition = alltrue([for r in var.allowed_ssh_ip_ranges : can(cidrhost(r, 32))]) + error_message = "Each element of var.allowed_ssh_ip_ranges must be a valid CIDR-formatted IPv4 range." + } +} + +variable "firewall_rules" { + type = any + description = "List of firewall rules" + default = [] +} + +variable "firewall_log_config" { + type = string + description = "Firewall log configuration for Toolkit firewall rules (var.enable_iap_ssh_ingress and others)" + default = "DISABLE_LOGGING" + nullable = false + + validation { + condition = contains([ + "INCLUDE_ALL_METADATA", + "EXCLUDE_ALL_METADATA", + "DISABLE_LOGGING", + ], var.firewall_log_config) + error_message = "var.firewall_log_config must be set to \"DISABLE_LOGGING\", or enable logging with \"INCLUDE_ALL_METADATA\" or \"EXCLUDE_ALL_METADATA\"" + } +} + +variable "network_profile" { + type = string + description = <<-EOT + A full or partial URL of the network profile to apply to this network. + This field can be set only at resource creation time. For example, the + following are valid URLs: + - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} + - projects/{projectId}/global/networkProfiles/{network_profile_name}} + When using a Mellanox network profile (contains 'roce'), if firewall_rules is specified or enable_internal_traffic is true, an error will be thrown + EOT + default = null +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/vpc/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/network/vpc/versions.tf new file mode 100644 index 0000000000..71b7106734 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/network/vpc/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 0.15.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/README.md b/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/README.md new file mode 100644 index 0000000000..192d7575a5 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/README.md @@ -0,0 +1,320 @@ +# Custom Images in the Cluster Toolkit (formerly HPC Toolkit) + +Please review the +[introduction to image building](../../../docs/image-building.md) for general +information on building custom images using the Toolkit. + +## Introduction + +This module uses [Packer](https://www.packer.io/) to create an image within an +Cluster Toolkit deployment. Packer operates by provisioning a short-lived VM in +Google Cloud on which it executes scripts to customize the boot disk for +repeated use. The VM's boot disk is specified from a source image that defaults +to the [HPC VM Image][hpcimage]. This Packer "template" supports customization +by the following approaches following a [recommended use](#recommended-use): + +- [startup-script metadata][startup-metadata] from [raw string][sss] or + [file][ssf] +- [Shell scripts][shell] uploaded from the Packer execution environment to the + VM +- [Ansible playbooks][ansible] uploaded from the Packer execution environment to + the VM + +They can be specified independently of one another, so that anywhere from 1 to 3 +solutions can be used simultaneously. In the case that 0 scripts are supplied, +the source boot disk is effectively copied to your project without +customization. This can be useful in scenarios where increased control over the +image maintenance lifecycle is desired or when policies restrict the use of +images to internal projects. + +## Minimum requirements + +### Outbound internet access + +Most customization scripts require access to resources on the public internet. +This can be achieved by one of the following 2 approaches: + +1. Using a public IP address on the VM + +- Set [var.omit_external_ip](#input_omit_external_ip) to `false` + +1. Configuring a VPC with a Cloud NAT in the region of the VM + +- Use the [vpc] module which automates NAT creation + +### Inbound internet access + +Read [order of execution](#order-of-execution) below for a discussion of VM +customization solutions and their requirements for inbound SSH access. +[Environments without SSH access](#environments-without-ssh-access) should use +the metadata-based startup-script solution. + +A simple way to enable inbound SSH access is to use the VPC module with +`allowed_ssh_ip_ranges` set to `0.0.0.0/0`. + +### User or service account executing Packer at command line + +The user or service account running Packer must have the permission to create +VMs in the selected VPC network and, if [use\_iap](#input_use_iap) is set, must +have the "IAP-Secured Tunnel User" role. Recommended roles are: + +- `roles/compute.instanceAdmin.v1` +- `roles/iap.tunnelResourceAccessor` + +### VM service account roles + +The service account attached to the temporary build VM created by Packer should +have the ability to write Cloud Logging entries so that you may inspect and +debug build logs. When using the metadata startup-script customization solution, +the service account attached to the temporary build VM created by Packer must +have the permission to modify its own metadata and to read from Cloud Storage +buckets. Recommended roles are: + +- `roles/compute.instanceAdmin.v1` +- `roles/iam.serviceAccountUser` +- `roles/logging.logWriter` +- `roles/monitoring.metricWriter` +- `roles/storage.objectViewer` + +It is recommended to create this service account as a separate step outside a +blueprint due to known delay in [IAM bindings propagation][iamprop]. + +## Example blueprints + +A recommended pattern for building images with this module is to use the +terraform based [startup-script] module along with this packer custom-image +module. Below you can find links to several examples of this pattern, including +usage instructions. + +### [Image Builder] + +The [Image Builder] blueprint demonstrates a solution that builds an image +using: + +- The [HPC VM Image][hpcimage] as a base upon which to customize +- A VPC network with firewall rules that allow IAP-based SSH tunnels +- A Toolkit runner that installs a custom script + +Please review the [examples README] for usage instructions. + +## Order of execution + +The startup script specified in metadata executes in parallel with the other +supported methods. However, the remaining methods execute in a well-defined +order relative to one another. + +1. All shell scripts will execute in the configured order +1. After shell scripts complete, all Ansible playbooks will execute in the + configured order + +> **_NOTE:_** if both [startup_script][sss] and [startup_script_file][ssf] are +> specified, then [startup_script_file][ssf] takes precedence. + +## Recommended use + +Because the [metadata startup script executes in parallel](#order-of-execution) +with the other solutions, conflicts can arise, especially when package managers +(`yum` or `apt`) lock their databases during package installation. Therefore, it +is recommended to choose one of the following approaches: + +1. Specify _either_ [startup_script][sss] _or_ [startup_script_file][ssf] and do + not specify [shell_scripts][shell] or [ansible_playbooks][ansible]. + - This can be especially useful in + [environments that restrict SSH access](#environments-without-ssh-access) +1. Specify any combination of [shell_scripts][shell] and + [ansible_playbooks][ansible] and do not specify [startup_script][sss] or + [startup_script_file][ssf]. + +If any of the startup script approaches fail by returning a code other than 0, +Packer will determine that the build has failed and refuse to save the image. + +## External access with SSH + +The [shell scripts][shell] and [Ansible playbooks][ansible] customization +solutions both require SSH access to the VM from the Packer execution +environment. SSH access can be enabled one of 2 ways: + +1. The VM is created without a public IP address and SSH tunnels are created + using [Identity-Aware Proxy (IAP)][iaptunnel]. + - Allow [use_iap](#input_use_iap) to take on its default value of `true` +1. The VM is created with an IP address on the public internet and firewall + rules allow SSH access from the Packer execution environment. + - Set `omit_external_ip = false` (or `omit_external_ip: false` in a + blueprint) + - Add firewall rules that open SSH to the VM + +The Packer template defaults to using to the 1st IAP-based solution because it +is more secure (no exposure to public internet) and because the [vpc] module +automatically sets up all necessary firewall rules for SSH tunneling and +outbound-only access to the internet through [Cloud NAT][cloudnat]. + +In either SSH solution, customization scripts should be supplied as files in the +[shell_scripts][shell] and [ansible_playbooks][ansible] settings. + +## Environments without SSH access + +Many network environments disallow SSH access to VMs. In these environments, the +[metadata-based startup scripts][startup-metadata] are appropriate because they +execute entirely independently of the Packer execution environment. + +In this scenario, a single scripts should be supplied in the form of a string to +the [startup_script][sss] input variable. This solution integrates well with +Toolkit runners. Runners operate by using a single startup script whose behavior +is extended by downloading and executing a customizable set of runners from +Cloud Storage at startup. + +> **_NOTE:_** Packer will attempt to use SSH if either [shell_scripts][shell] or +> [ansible_playbooks][ansible] are set to non-empty values. Leave them at their +> default, empty values to ensure access by SSH is disabled. + +## Supplying startup script as a string + +The [startup_script][sss] parameter accepts scripts formatted as strings. In +Packer and Terraform, multi-line strings can be specified using +[heredoc syntax](https://www.terraform.io/language/expressions/strings#heredoc-strings) +in an input [Packer variables file][pkrvars] (`*.pkrvars.hcl`) For example, the +following snippet defines a multi-line bash script followed by an integer +representing the size, in GiB, of the resulting image: + +```hcl +startup_script = <<-EOT + #!/bin/bash + yum install -y epel-release + yum install -y jq + EOT + +disk_size = 100 +``` + +In a blueprint, the equivalent syntax is: + +```yaml +... + settings: + startup_script: | + #!/bin/bash + yum install -y epel-release + yum install -y jq + disk_size: 100 +... +``` + +## Monitoring startup script execution + +When using startup script customization, Packer will print very limited output +to the console. For example: + +```text +==> example.googlecompute.toolkit_image: Waiting for any running startup script to finish... +==> example.googlecompute.toolkit_image: Startup script not finished yet. Waiting... +==> example.googlecompute.toolkit_image: Startup script not finished yet. Waiting... +==> example.googlecompute.toolkit_image: Startup script, if any, has finished running. +``` + +### Debugging startup-script failures + +> [!NOTE] +> There can be a delay in the propagation of the logs from the instance to +> Cloud Logging, so it may require waiting a few minutes to see the full logs. + +If the Packer image build fails, the module will output a `gcloud` command +that can be used directly to review startup-script execution. + +## License + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at + +```text + http://www.apache.org/licenses/LICENSE-2.0 +``` + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. + + +## Requirements + +No requirements. + +## Providers + +No providers. + +## Modules + +No modules. + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [accelerator\_count](#input\_accelerator\_count) | Number of accelerator cards to attach to the VM; not necessary for families that always include GPUs (A2). | `number` | `null` | no | +| [accelerator\_type](#input\_accelerator\_type) | Type of accelerator cards to attach to the VM; not necessary for families that always include GPUs (A2). | `string` | `null` | no | +| [ansible\_playbooks](#input\_ansible\_playbooks) | A list of Ansible playbook configurations that will be uploaded to customize the VM image | 
list(object({
    playbook_file   = string
    galaxy_file     = string
    extra_arguments = list(string)
  }))
| `[]` | no | +| [communicator](#input\_communicator) | Communicator to use for provisioners that require access to VM ("ssh" or "winrm") | `string` | `null` | no | +| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name | `string` | n/a | yes | +| [disk\_size](#input\_disk\_size) | Size of disk image in GB | `number` | `null` | no | +| [disk\_type](#input\_disk\_type) | Type of persistent disk to provision | `string` | `"pd-balanced"` | no | +| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | +| [image\_family](#input\_image\_family) | The family name of the image to be built. Defaults to `deployment_name` | `string` | `null` | no | +| [image\_name](#input\_image\_name) | The name of the image to be built. If not supplied, it will be set to image\_family-$ISO\_TIMESTAMP | `string` | `null` | no | +| [image\_storage\_locations](#input\_image\_storage\_locations) | Storage location, either regional or multi-regional, where snapshot content is to be stored and only accepts 1 value.
See https://developer.hashicorp.com/packer/plugins/builders/googlecompute#image_storage_locations | `list(string)` | `null` | no | +| [labels](#input\_labels) | Labels to apply to the short-lived VM | `map(string)` | `null` | no | +| [machine\_type](#input\_machine\_type) | VM machine type on which to build new image | `string` | `"n2-standard-4"` | no | +| [manifest\_file](#input\_manifest\_file) | File to which to write Packer build manifest | `string` | `"packer-manifest.json"` | no | +| [metadata](#input\_metadata) | Instance metadata for the builder VM (use var.startup\_script or var.startup\_script\_file to set startup-script metadata) | `map(string)` | `{}` | no | +| [network\_project\_id](#input\_network\_project\_id) | Project ID of Shared VPC network | `string` | `null` | no | +| [omit\_external\_ip](#input\_omit\_external\_ip) | Provision the image building VM without a public IP address | `bool` | `true` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE` | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which to create VM and image | `string` | n/a | yes | +| [scopes](#input\_scopes) | DEPRECATED: use var.service\_account\_scopes | `set(string)` | `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | The service account email to use. If null or 'default', then the default Compute Engine service account will be used. | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Service account scopes to attach to the instance. See
https://cloud.google.com/compute/docs/access/service-accounts. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [shell\_scripts](#input\_shell\_scripts) | A list of paths to local shell scripts which will be uploaded to customize the VM image | `list(string)` | `[]` | no | +| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | +| [source\_image](#input\_source\_image) | Source OS image to build from | `string` | `null` | no | +| [source\_image\_family](#input\_source\_image\_family) | Alternative to source\_image. Specify image family to build from latest image in family | `string` | `"hpc-rocky-linux-8"` | no | +| [source\_image\_project\_id](#input\_source\_image\_project\_id) | A list of project IDs to search for the source image. Packer will search the
first project ID in the list first, and fall back to the next in the list,
until it finds the source image. | `list(string)` | `null` | no | +| [ssh\_username](#input\_ssh\_username) | Username to use for SSH access to VM | `string` | `"hpc-toolkit-packer"` | no | +| [startup\_script](#input\_startup\_script) | Startup script (as raw string) used to build the custom Linux VM image (overridden by var.startup\_script\_file if both are set) | `string` | `null` | no | +| [startup\_script\_file](#input\_startup\_script\_file) | File path to local shell script that will be used to customize the Linux VM image (overrides var.startup\_script) | `string` | `null` | no | +| [state\_timeout](#input\_state\_timeout) | The time to wait for instance state changes, including image creation | `string` | `"10m"` | no | +| [subnetwork\_name](#input\_subnetwork\_name) | Name of subnetwork in which to provision image building VM | `string` | n/a | yes | +| [tags](#input\_tags) | Assign network tags to apply firewall rules to VM instance | `list(string)` | `null` | no | +| [use\_iap](#input\_use\_iap) | Use IAP proxy when connecting by SSH | `bool` | `true` | no | +| [use\_os\_login](#input\_use\_os\_login) | Use OS Login when connecting by SSH | `bool` | `false` | no | +| [windows\_startup\_ps1](#input\_windows\_startup\_ps1) | A list of strings containing PowerShell scripts which will customize a Windows VM image (requires WinRM communicator) | `list(string)` | `[]` | no | +| [wrap\_startup\_script](#input\_wrap\_startup\_script) | Wrap startup script with Packer-generated wrapper | `bool` | `true` | no | +| [zone](#input\_zone) | Cloud zone in which to provision image building VM | `string` | n/a | yes | + +## Outputs + +No outputs. + + +[ansible]: #input_ansible_playbooks +[cloudnat]: https://cloud.google.com/nat/docs/overview +[examples readme]: ../../../examples/README.md#image-builderyaml- +[hpcimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm +[iamprop]: https://cloud.google.com/iam/docs/access-change-propagation +[iaptunnel]: https://cloud.google.com/iap/docs/using-tcp-forwarding +[image builder]: ../../../examples/image-builder.yaml +[logging-console]: https://console.cloud.google.com/logs/ +[logging-read-docs]: https://cloud.google.com/sdk/gcloud/reference/logging/read +[pkrvars]: https://www.packer.io/guides/hcl/variables#from-a-file +[shell]: #input_shell_scripts +[ssf]: #input_startup_script_file +[sss]: #input_startup_script +[startup-metadata]: https://cloud.google.com/compute/docs/instances/startup-scripts/linux +[startup-script]: ../../../modules/scripts/startup-script +[vpc]: ../../network/vpc/README.md diff --git a/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/image.pkr.hcl b/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/image.pkr.hcl new file mode 100644 index 0000000000..9282cf7433 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/image.pkr.hcl @@ -0,0 +1,216 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "custom-image", ghpc_role = "packer" }) + + # construct a unique image name from the image family + image_family = var.image_family != null ? var.image_family : var.deployment_name + image_name_default = "${local.image_family}-${formatdate("YYYYMMDD't'hhmmss'z'", timestamp())}" + image_name = var.image_name != null ? var.image_name : local.image_name_default + + # construct vm image name for use when getting logs + instance_name = "packer-${substr(uuidv4(), 0, 6)}" + + # default to explicit var.communicator, otherwise in-order: ssh/winrm/none + shell_script_communicator = length(var.shell_scripts) > 0 ? "ssh" : "" + ansible_playbook_communicator = length(var.ansible_playbooks) > 0 ? "ssh" : "" + powershell_script_communicator = length(var.windows_startup_ps1) > 0 ? "winrm" : "" + communicator = coalesce( + var.communicator, + local.shell_script_communicator, + local.ansible_playbook_communicator, + local.powershell_script_communicator, + "none" + ) + + # must not enable IAP when no communicator is in use + use_iap = local.communicator == "none" ? false : var.use_iap + + # construct metadata from startup_script and metadata variables + startup_script_metadata = var.startup_script == null ? {} : { startup-script = var.startup_script } + + linux_user_metadata = { + block-project-ssh-keys = "TRUE" + shutdown-script = <<-EOT + #!/bin/bash + userdel -r ${var.ssh_username} + sed -i '/${var.ssh_username}/d' /var/lib/google/google_users + EOT + } + windows_packer_user = "packer_user" + windows_user_metadata = { + sysprep-specialize-script-cmd = "winrm quickconfig -quiet & net user /add ${local.windows_packer_user} & net localgroup administrators ${local.windows_packer_user} /add & winrm set winrm/config/service/auth @{Basic=\\\"true\\\"}" + windows-shutdown-script-cmd = <<-EOT + net user /delete ${local.windows_packer_user} + EOT + } + user_metadata = local.communicator == "winrm" ? local.windows_user_metadata : local.linux_user_metadata + + # merge metadata such that var.metadata always overrides user management + # metadata but always allow var.startup_script to override var.metadata + metadata = merge( + local.user_metadata, + var.metadata, + local.startup_script_metadata, + ) + + # determine best value for on_host_maintenance if not supplied by user + machine_vals = split("-", var.machine_type) + machine_family = local.machine_vals[0] + gpu_attached = contains(["a2", "g2"], local.machine_family) || var.accelerator_type != null + on_host_maintenance_default = local.gpu_attached ? "TERMINATE" : "MIGRATE" + on_host_maintenance = ( + var.on_host_maintenance != null + ? var.on_host_maintenance + : local.on_host_maintenance_default + ) + + accelerator_type = var.accelerator_type == null ? null : "projects/${var.project_id}/zones/${var.zone}/acceleratorTypes/${var.accelerator_type}" + + winrm_username = local.communicator == "winrm" ? "packer_user" : null + winrm_insecure = local.communicator == "winrm" ? true : null + winrm_use_ssl = local.communicator == "winrm" ? true : null + + enable_integrity_monitoring = var.enable_shielded_vm && var.shielded_instance_config.enable_integrity_monitoring + enable_secure_boot = var.enable_shielded_vm && var.shielded_instance_config.enable_secure_boot + enable_vtpm = var.enable_shielded_vm && var.shielded_instance_config.enable_vtpm + + image_licenses = [ + "projects/click-to-deploy-images/global/licenses/hpc-toolkit-vm-image" + ] +} + +source "googlecompute" "toolkit_image" { + communicator = local.communicator + project_id = var.project_id + image_name = local.image_name + image_family = local.image_family + image_labels = local.labels + instance_name = local.instance_name + machine_type = var.machine_type + accelerator_type = local.accelerator_type + accelerator_count = var.accelerator_count + on_host_maintenance = local.on_host_maintenance + disk_size = var.disk_size + disk_type = var.disk_type + omit_external_ip = var.omit_external_ip + use_internal_ip = var.omit_external_ip + subnetwork = var.subnetwork_name + network_project_id = var.network_project_id + service_account_email = var.service_account_email + scopes = var.service_account_scopes + source_image = var.source_image + source_image_family = var.source_image_family + source_image_project_id = var.source_image_project_id + ssh_username = var.ssh_username + tags = var.tags + use_iap = local.use_iap + use_os_login = var.use_os_login + winrm_username = local.winrm_username + winrm_insecure = local.winrm_insecure + winrm_use_ssl = local.winrm_use_ssl + zone = var.zone + labels = local.labels + metadata = local.metadata + startup_script_file = var.startup_script_file + wrap_startup_script = var.wrap_startup_script + state_timeout = var.state_timeout + image_storage_locations = var.image_storage_locations + enable_secure_boot = local.enable_secure_boot + enable_vtpm = local.enable_vtpm + enable_integrity_monitoring = local.enable_integrity_monitoring + image_licenses = local.image_licenses +} + +build { + name = var.deployment_name + sources = ["sources.googlecompute.toolkit_image"] + + # using dynamic blocks to create provisioners ensures that there are no + # provisioner blocks when none are provided and we can use the none + # communicator when using startup-script + + # provisioner "shell" blocks + dynamic "provisioner" { + labels = ["shell"] + for_each = var.shell_scripts + content { + execute_command = "sudo -H sh -c '{{ .Vars }} {{ .Path }}'" + script = provisioner.value + } + } + + # provisioner "powershell" blocks + dynamic "provisioner" { + labels = ["powershell"] + for_each = var.windows_startup_ps1 + content { + inline = split("\n", provisioner.value) + } + } + + dynamic "provisioner" { + labels = ["powershell"] + for_each = length(var.windows_startup_ps1) > 0 ? [1] : [] + content { + inline = [ + "GCESysprep -no_shutdown" + ] + } + } + + # provisioner "ansible-local" blocks + # this installs custom roles/collections from ansible-galaxy in /home/packer + # which will be removed at the end; consider modifying /etc/ansible/ansible.cfg + dynamic "provisioner" { + labels = ["ansible-local"] + for_each = var.ansible_playbooks + content { + playbook_file = provisioner.value.playbook_file + galaxy_file = provisioner.value.galaxy_file + extra_arguments = provisioner.value.extra_arguments + } + } + + post-processor "manifest" { + output = var.manifest_file + strip_path = true + custom_data = { + built-by = "cloud-hpc-toolkit" + } + } + + # If there is an error during image creation, print out command for getting packer VM logs + error-cleanup-provisioner "shell-local" { + environment_vars = [ + "PRJ_ID=${var.project_id}", + "INST_NAME=${local.instance_name}", + "ZONE=${var.zone}", + ] + inline_shebang = "/bin/bash -e" + inline = [ + "type -P gcloud > /dev/null || exit 0", + "INST_ID=$(gcloud compute instances describe $INST_NAME --project $PRJ_ID --format=\"value(id)\" --zone=$ZONE)", + "echo 'Error building image try checking logs:'", + join(" ", ["echo \"gcloud logging --project $PRJ_ID read", + "'logName=(\\\"projects/$PRJ_ID/logs/GCEMetadataScripts\\\" OR \\\"projects/$PRJ_ID/logs/google_metadata_script_runner\\\") AND resource.labels.instance_id=$INST_ID'", + "--format=\\\"table(timestamp, resource.labels.instance_id, jsonPayload.message)\\\"", + "--order=asc\"" + ] + ) + ] + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/metadata.yaml new file mode 100644 index 0000000000..23108c4e17 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com + - logging.googleapis.com + - storage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/variables.pkr.hcl b/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/variables.pkr.hcl new file mode 100644 index 0000000000..3cede102ce --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/variables.pkr.hcl @@ -0,0 +1,276 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "deployment_name" { + description = "Cluster Toolkit deployment name" + type = string +} + +variable "project_id" { + description = "Project in which to create VM and image" + type = string +} + +variable "machine_type" { + description = "VM machine type on which to build new image" + type = string + default = "n2-standard-4" +} + +variable "disk_size" { + description = "Size of disk image in GB" + type = number + default = null +} + +variable "disk_type" { + description = "Type of persistent disk to provision" + type = string + default = "pd-balanced" +} + +variable "zone" { + description = "Cloud zone in which to provision image building VM" + type = string +} + +variable "network_project_id" { + description = "Project ID of Shared VPC network" + type = string + default = null +} + +variable "subnetwork_name" { + description = "Name of subnetwork in which to provision image building VM" + type = string +} + +variable "omit_external_ip" { + description = "Provision the image building VM without a public IP address" + type = bool + default = true +} + +variable "tags" { + description = "Assign network tags to apply firewall rules to VM instance" + type = list(string) + default = null +} + +variable "image_family" { + description = "The family name of the image to be built. Defaults to `deployment_name`" + type = string + default = null +} + +variable "image_name" { + description = "The name of the image to be built. If not supplied, it will be set to image_family-$ISO_TIMESTAMP" + type = string + default = null +} + +variable "source_image_project_id" { + description = < +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.1 | +| [google](#requirement\_google) | >= 4.0 | +| [local](#requirement\_local) | >= 2.0.0 | +| [null](#requirement\_null) | ~> 3.0 | +| [random](#requirement\_random) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.0 | +| [local](#provider\_local) | >= 2.0.0 | +| [null](#provider\_null) | ~> 3.0 | +| [random](#provider\_random) | >= 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [instance\_template](#module\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | +| [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | ../../scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [local_file.job_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [local_file.submit_script](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [null_resource.submit_job](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [random_id.submit_job_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment, used for the job\_id | `string` | n/a | yes | +| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true, instances will have public IPs | `bool` | `true` | no | +| [gcloud\_version](#input\_gcloud\_version) | The version of the gcloud cli being used. Used for output instructions. Valid inputs are `"alpha"`, `"beta"` and "" (empty string for default version) | `string` | `""` | no | +| [image](#input\_image) | DEPRECATED: Google Cloud Batch compute node image. Ignored if `instance_template` is provided. | `any` | `null` | no | +| [instance\_image](#input\_instance\_image) | Google Cloud Batch compute node image. Ignored if `instance_template` is provided.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | +| [instance\_template](#input\_instance\_template) | Compute VM instance template self-link to be used for Google Cloud Batch compute node. If provided, a number of other variables will be ignored as noted by `Ignored if instance_template is provided` in descriptions. | `string` | `null` | no | +| [job\_filename](#input\_job\_filename) | The filename of the generated job template file. Will default to `cloud-batch-.json` if not specified | `string` | `null` | no | +| [job\_id](#input\_job\_id) | An id for the Google Cloud Batch job. Used for output instructions and file naming. Automatically populated by the module id if not set. If setting manually, ensure a unique value across all jobs. | `string` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the Google Cloud Batch compute nodes. Key-value pairs. Ignored if `instance_template` is provided. | `map(string)` | n/a | yes | +| [log\_policy](#input\_log\_policy) | Create a block to define log policy.
When set to `CLOUD_LOGGING`, logs will be sent to Cloud Logging.
When set to `PATH`, path must be added to generated template.
When set to `DESTINATION_UNSPECIFIED`, logs will not be preserved. | `string` | `"CLOUD_LOGGING"` | no | +| [machine\_type](#input\_machine\_type) | Machine type to use for Google Cloud Batch compute nodes. Ignored if `instance_template` is provided. | `string` | `"n2-standard-4"` | no | +| [mpi\_mode](#input\_mpi\_mode) | Sets up barriers before and after each runnable. In addition, sets `permissiveSsh=true`, `requireHostsFile=true`, and `taskCountPerNode=1`. `taskCountPerNode` can be overridden by `task_count_per_node`. | `bool` | `false` | no | +| [native\_batch\_mounting](#input\_native\_batch\_mounting) | Batch can mount some fs\_type nativly using the 'volumes' block in the job file. If set to false, all mounting will happen through Cluster Toolkit startup scripts. | `bool` | `true` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. Ignored if `instance_template` is provided. | 
list(object({
    server_ip             = string
    remote_mount          = string
    local_mount           = string
    fs_type               = string
    mount_options         = string
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE` | `string` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The region in which to run the Google Cloud Batch job | `string` | n/a | yes | +| [runnable](#input\_runnable) | A simplified form of `var.runnables` that only takes a single script. Use either `runnables` or `runnable`. | `string` | `null` | no | +| [runnables](#input\_runnables) | A list of shell scripts to be executed in sequence as the main workload of the Google Batch job. These will be used to populate the generated template. | 
list(object({
    script = string
  }))
| `null` | no | +| [service\_account](#input\_service\_account) | Service account to attach to the Google Cloud Batch compute node. Ignored if `instance_template` is provided. | 
object({
    email  = string,
    scopes = set(string)
  })
| 
{
  "email": null,
  "scopes": [
    "https://www.googleapis.com/auth/devstorage.read_only",
    "https://www.googleapis.com/auth/logging.write",
    "https://www.googleapis.com/auth/monitoring.write",
    "https://www.googleapis.com/auth/servicecontrol",
    "https://www.googleapis.com/auth/service.management.readonly",
    "https://www.googleapis.com/auth/trace.append"
  ]
}
| no | +| [startup\_script](#input\_startup\_script) | Startup script run before Google Cloud Batch job starts. Ignored if `instance_template` is provided. | `string` | `null` | no | +| [submit](#input\_submit) | When set to true, the generated job file will be submitted automatically to Google Cloud as part of terraform apply. | `bool` | `false` | no | +| [subnetwork](#input\_subnetwork) | The subnetwork that the Batch job should run on. Defaults to 'default' subnet. Ignored if `instance_template` is provided. | `any` | `null` | no | +| [task\_count](#input\_task\_count) | Number of parallel tasks | `number` | `1` | no | +| [task\_count\_per\_node](#input\_task\_count\_per\_node) | Max number of tasks that can be run on a VM at the same time. If not specified, Batch will decide a value. | `number` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [gcloud\_version](#output\_gcloud\_version) | The version of gcloud to be used. | +| [instance\_template](#output\_instance\_template) | Instance template used by the Batch job. | +| [instructions](#output\_instructions) | Instructions for submitting the Batch job. | +| [job\_data](#output\_job\_data) | All data associated with the defined job, typically provided as input to clout-batch-login-node. | +| [network\_storage](#output\_network\_storage) | An array of network attached storage mounts used by the Batch job. | +| [startup\_script](#output\_startup\_script) | Startup script run before Google Cloud Batch job starts. | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf new file mode 100644 index 0000000000..7a7fe02307 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_compute_image" "compute_image" { + family = try(var.instance_image.family, null) + name = try(var.instance_image.name, null) + project = try(var.instance_image.project, null) + + lifecycle { + postcondition { + # Condition needs to check the suffix of the license, as prefix contains an API version which can change. + # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates + condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) + error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" + } + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/main.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/main.tf new file mode 100644 index 0000000000..0d681536c9 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/main.tf @@ -0,0 +1,149 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "batch-job-template", ghpc_role = "scheduler" }) +} + +locals { + instance_template = coalesce(var.instance_template, module.instance_template.self_link) + + tasks_per_node = var.task_count_per_node != null ? var.task_count_per_node : (var.mpi_mode ? 1 : null) + + one_line_runnable = coalesce(var.runnable, "## Add your workload here ##") + runnables = coalesce(var.runnables, [{ script = local.one_line_runnable }]) + + job_template_contents = templatefile( + "${path.module}/templates/batch-job-base.yaml.tftpl", + { + synchronized = var.mpi_mode + runnables = local.runnables + task_count = var.task_count + tasks_per_node = local.tasks_per_node + require_hosts_file = var.mpi_mode + permissive_ssh = var.mpi_mode + log_policy = var.log_policy + instance_template = local.instance_template + nfs_volumes = local.native_batch_network_storage + labels = local.labels + } + ) + + submit_job_id = "${var.job_id}-${random_id.submit_job_suffix.hex}" + job_filename = coalesce(var.job_filename, "${var.job_id}.yaml") + job_template_output_path = "${path.root}/${local.job_filename}" + + submit_script_contents = templatefile( + "${path.module}/templates/batch-submit.sh.tftpl", + { + project = var.project_id + location = var.region + config = local_file.job_template.filename + submit_job_id = local.submit_job_id + } + ) + submit_script_output_path = "${path.root}/submit-${var.job_id}.sh" + + subnetwork_name = var.subnetwork != null ? var.subnetwork.name : "default" + subnetwork_project = var.subnetwork != null ? var.subnetwork.project : var.project_id + + # Filter network_storage for native Batch support + native_fstype = var.native_batch_mounting ? ["nfs"] : [] + native_batch_network_storage = [ + for ns in var.network_storage : + ns if contains(local.native_fstype, ns.fs_type) + ] + # other processing happens in startup_from_network_storage.tf + + # this code is similar to code in Packer and vm-instance modules + # it differs in that this module does not (yet) expose var.guest_acclerator + # for attaching GPUs to N1 VMs. For now, identify only A2 types. + machine_vals = split("-", var.machine_type) + machine_family = local.machine_vals[0] + gpu_attached = contains(["a2", "g2"], local.machine_family) + on_host_maintenance_default = local.gpu_attached ? "TERMINATE" : "MIGRATE" + + on_host_maintenance = coalesce(var.on_host_maintenance, local.on_host_maintenance_default) + + network_storage_metadata = var.network_storage != null ? ({ network_storage = jsonencode(var.network_storage) }) : {} + disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } + + metadata = merge( + local.network_storage_metadata, + local.disable_automatic_updates_metadata + ) +} + +module "instance_template" { + source = "terraform-google-modules/vm/google//modules/instance_template" + version = "~> 12.1" + + name_prefix = var.instance_template == null ? "${var.job_id}-instance-template" : "unused-template" + project_id = var.project_id + subnetwork = local.subnetwork_name + subnetwork_project = local.subnetwork_project + service_account = var.service_account + access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] + labels = local.labels + + machine_type = var.machine_type + startup_script = local.startup_from_network_storage + metadata = local.metadata + source_image_family = data.google_compute_image.compute_image.family + source_image = data.google_compute_image.compute_image.name + source_image_project = data.google_compute_image.compute_image.project + on_host_maintenance = local.on_host_maintenance +} + +resource "local_file" "job_template" { + content = local.job_template_contents + filename = local.job_template_output_path + + lifecycle { + precondition { + condition = var.runnable == null || var.runnables == null + error_message = "var.runnable and var.runnables (plural) cannot both be set." + } + } +} + +resource "random_id" "submit_job_suffix" { + byte_length = 4 + keepers = { + always_run = timestamp() + } +} + +resource "local_file" "submit_script" { + content = local.submit_script_contents + filename = local.submit_script_output_path +} + +resource "null_resource" "submit_job" { + depends_on = [local_file.job_template, local_file.submit_script] + count = var.submit ? 1 : 0 + + # A new deployment should always submit a new job. Old finished jobs aren't persistent parts of + # Cloud infrastructure. + triggers = { + always_run = timestamp() + } + + provisioner "local-exec" { + command = local.submit_script_output_path + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml new file mode 100644 index 0000000000..387e810962 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml @@ -0,0 +1,22 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - batch.googleapis.com + - compute.googleapis.com +ghpc: + inject_module_id: job_id diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/outputs.tf new file mode 100644 index 0000000000..0b1295975a --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/outputs.tf @@ -0,0 +1,80 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + provided_instance_tpl_msg = "The Batch job template uses the existing VM instance template:" + generated_instance_tpl_msg = "The Batch job template uses a new VM instance template created matching the provided settings:" + submit_msg = <<-EOT + + The job has been submitted. See job status at: + https://console.cloud.google.com/batch/jobsDetail/regions/${var.region}/jobs/${local.submit_job_id}?project=${var.project_id} + EOT +} + +output "instructions" { + description = "Instructions for submitting the Batch job." + value = <<-EOT + + A Batch job template file has been created locally at: + ${abspath(local.job_template_output_path)} + + ${var.instance_template == null ? local.generated_instance_tpl_msg : local.provided_instance_tpl_msg} + ${local.instance_template} + ${var.submit ? local.submit_msg : ""} + + Use the following commands to: + Submit your job${var.submit ? " (Note: job has already been submitted)" : ""}: + gcloud ${var.gcloud_version} batch jobs submit ${local.submit_job_id} --config=${abspath(local.job_template_output_path)} --location=${var.region} --project=${var.project_id} + + Check status: + gcloud ${var.gcloud_version} batch jobs describe ${local.submit_job_id} --location=${var.region} --project=${var.project_id} | grep state: + + Delete job: + gcloud ${var.gcloud_version} batch jobs delete ${local.submit_job_id} --location=${var.region} --project=${var.project_id} + + List all jobs: + gcloud ${var.gcloud_version} batch jobs list --project=${var.project_id} + EOT +} + +output "job_data" { + description = "All data associated with the defined job, typically provided as input to clout-batch-login-node." + value = { + template_contents = local.job_template_contents, + filename = local.job_filename, + id = local.submit_job_id + } +} + +output "instance_template" { + description = "Instance template used by the Batch job." + value = local.instance_template +} + +output "network_storage" { + description = "An array of network attached storage mounts used by the Batch job." + value = var.network_storage +} + +output "startup_script" { + description = "Startup script run before Google Cloud Batch job starts." + value = var.startup_script +} + +output "gcloud_version" { + description = "The version of gcloud to be used." + value = var.gcloud_version +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf new file mode 100644 index 0000000000..02bc58e4f7 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# This file is meant to be reused by multiple modules. +# "inputs": +# local.native_fstype : list of file systems that are supported automatically, but looking at the metadata. +# var.network_storage : to be passed into metadata somewhere else (not here) +# var.startup_script : to be changed into a more complete file system with all the fs runners + +# "outputs": +# local.startup_from_network_storage : A full startup script with all the runners that are not supported +# natively and were included in the network_storage structure + +locals { + startup_script_network_storage = [ + for ns in var.network_storage : + ns if !contains(local.native_fstype, ns.fs_type) + ] + # Pull out runners to include in startup script + storage_client_install_runners = [ + for ns in local.startup_script_network_storage : + ns.client_install_runner if ns.client_install_runner != null + ] + mount_runners = [ + for ns in local.startup_script_network_storage : + ns.mount_runner if ns.mount_runner != null + ] + + startup_script_runner = [{ + content = var.startup_script != null ? var.startup_script : "echo 'No user provided startup script.'" + destination = "passed_startup_script.sh" + type = "shell" + }] + + full_runner_list = concat( + local.storage_client_install_runners, + local.mount_runners, + local.startup_script_runner + ) + + startup_from_network_storage = module.netstorage_startup_script.startup_script +} + +module "netstorage_startup_script" { + source = "../../scripts/startup-script" + + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = local.full_runner_list +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl new file mode 100644 index 0000000000..83fccde53b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl @@ -0,0 +1,53 @@ +taskGroups: + - taskSpec: + runnables: + %{~ if synchronized ~} + - barrier: + name: "wait-for-node-startup" + %{~ endif ~} + %{~ for runnable in runnables ~} + - script: + text: ${indent(12, chomp(yamlencode(runnable.script)))} + %{~ if synchronized ~} + - barrier: + name: "wait-for-script-to-complete" + %{~ endif ~} + %{~ endfor ~} + %{~ if length(nfs_volumes) > 0 ~} + volumes: + %{~ for index, vol in nfs_volumes ~} + - nfs: + server: "${vol.server_ip}" + remotePath: "${vol.remote_mount}" + %{~ if vol.mount_options != "" && vol.mount_options != null ~} + mountOptions: "${vol.mount_options}" + %{~ endif ~} + mountPath: "${vol.local_mount}" + %{~ endfor ~} + %{~ endif ~} + taskCount: ${task_count} + %{~ if tasks_per_node != null ~} + taskCountPerNode: ${tasks_per_node} + %{~ endif ~} + requireHostsFile: ${require_hosts_file} + permissiveSsh: ${permissive_ssh} +%{~ if instance_template != null } +allocationPolicy: + instances: + - instanceTemplate: "${instance_template}" +%{~ endif } +%{~ if log_policy == "CLOUD_LOGGING" } +logsPolicy: + destination: "CLOUD_LOGGING" +%{ endif } +%{~ if log_policy == "PATH" } +logsPolicy: + destination: "PATH" + logsPath: ## Add logging path here +%{ endif } +%{~ if length(labels) > 0 ~} +labels: +%{ for k, v in labels ~} + ${k}: "${v}" +%{ endfor } +%{~ endif ~} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl new file mode 100644 index 0000000000..25f89c3ceb --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl @@ -0,0 +1,10 @@ +#!/bin/bash +set -e -o pipefail +GCLOUD_MAJOR_VERSION=$(gcloud --version | head -n 1 | awk '{print $NF}' | cut -f1 --delimiter=.) +if [ $((GCLOUD_MAJOR_VERSION >= 461)) ]; then + gcloud batch jobs submit ${submit_job_id} --project=${project} --location=${location} --config=${config} + echo "batch job ${submit_job_id} successfully submitted" +else + echo "gcloud must be updated to version 461.0.0 or later." + exit 1 +fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/variables.tf new file mode 100644 index 0000000000..f65fbd111e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/variables.tf @@ -0,0 +1,240 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "region" { + description = "The region in which to run the Google Cloud Batch job" + type = string +} + +variable "deployment_name" { + description = "Name of the deployment, used for the job_id" + type = string +} + +variable "labels" { + description = "Labels to add to the Google Cloud Batch compute nodes. Key-value pairs. Ignored if `instance_template` is provided." + type = map(string) +} + +variable "job_id" { + description = "An id for the Google Cloud Batch job. Used for output instructions and file naming. Automatically populated by the module id if not set. If setting manually, ensure a unique value across all jobs." + type = string +} + +variable "job_filename" { + description = "The filename of the generated job template file. Will default to `cloud-batch-.json` if not specified" + type = string + default = null +} + +variable "gcloud_version" { + description = "The version of the gcloud cli being used. Used for output instructions. Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default version)" + type = string + default = "" + + validation { + condition = contains(["alpha", "beta", ""], var.gcloud_version) + error_message = "Allowed values for gcloud_version are 'alpha', 'beta', or '' (empty string)." + } +} + +variable "task_count" { + description = "Number of parallel tasks" + type = number + default = 1 +} + +variable "task_count_per_node" { + description = "Max number of tasks that can be run on a VM at the same time. If not specified, Batch will decide a value." + type = number + default = null +} + +variable "mpi_mode" { + description = "Sets up barriers before and after each runnable. In addition, sets `permissiveSsh=true`, `requireHostsFile=true`, and `taskCountPerNode=1`. `taskCountPerNode` can be overridden by `task_count_per_node`." + type = bool + default = false +} + +variable "log_policy" { + description = <<-EOT + Create a block to define log policy. + When set to `CLOUD_LOGGING`, logs will be sent to Cloud Logging. + When set to `PATH`, path must be added to generated template. + When set to `DESTINATION_UNSPECIFIED`, logs will not be preserved. + EOT + type = string + default = "CLOUD_LOGGING" + + validation { + condition = contains(["CLOUD_LOGGING", "PATH", "DESTINATION_UNSPECIFIED"], var.log_policy) + error_message = "Allowed values for log_policy are 'CLOUD_LOGGING', 'PATH', or 'DESTINATION_UNSPECIFIED'." + } +} + +variable "runnables" { + description = "A list of shell scripts to be executed in sequence as the main workload of the Google Batch job. These will be used to populate the generated template." + type = list(object({ + script = string + })) + default = null +} + +variable "runnable" { + description = "A simplified form of `var.runnables` that only takes a single script. Use either `runnables` or `runnable`." + type = string + default = null +} + +variable "instance_template" { + description = "Compute VM instance template self-link to be used for Google Cloud Batch compute node. If provided, a number of other variables will be ignored as noted by `Ignored if instance_template is provided` in descriptions." + type = string + default = null +} + +variable "subnetwork" { + description = "The subnetwork that the Batch job should run on. Defaults to 'default' subnet. Ignored if `instance_template` is provided." + type = any + default = null +} + +variable "enable_public_ips" { + description = "If set to true, instances will have public IPs" + type = bool + default = true +} + +variable "service_account" { + description = "Service account to attach to the Google Cloud Batch compute node. Ignored if `instance_template` is provided." + type = object({ + email = string, + scopes = set(string) + }) + default = { + email = null + scopes = [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring.write", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/trace.append" + ] + } +} + +variable "machine_type" { + description = "Machine type to use for Google Cloud Batch compute nodes. Ignored if `instance_template` is provided." + type = string + default = "n2-standard-4" +} + +variable "startup_script" { + description = "Startup script run before Google Cloud Batch job starts. Ignored if `instance_template` is provided." + type = string + default = null +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured. Ignored if `instance_template` is provided." + type = list(object({ + server_ip = string + remote_mount = string + local_mount = string + fs_type = string + mount_options = string + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "native_batch_mounting" { + description = "Batch can mount some fs_type nativly using the 'volumes' block in the job file. If set to false, all mounting will happen through Cluster Toolkit startup scripts." + type = bool + default = true +} + +# Deprecated, replaced by instance_image +# tflint-ignore: terraform_unused_declarations +variable "image" { + description = "DEPRECATED: Google Cloud Batch compute node image. Ignored if `instance_template` is provided." + type = any + default = null + + validation { + condition = var.image == null + error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." + } +} + +variable "instance_image" { + description = <<-EOD + Google Cloud Batch compute node image. Ignored if `instance_template` is provided. + + Expected Fields: + name: The name of the image. Mutually exclusive with family. + family: The image family to use. Mutually exclusive with name. + project: The project where the image is hosted. + EOD + type = map(string) + default = { + project = "cloud-hpc-image-public" + family = "hpc-rocky-linux-8" + } + + validation { + condition = can(coalesce(var.instance_image.project)) + error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." + } + + validation { + condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) + error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." + } +} + +variable "on_host_maintenance" { + description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE`" + type = string + default = null + validation { + condition = var.on_host_maintenance == null ? true : contains(["MIGRATE", "TERMINATE"], var.on_host_maintenance) + error_message = "When set, the on_host_maintenance must be set to MIGRATE or TERMINATE." + } +} + +variable "submit" { + description = "When set to true, the generated job file will be submitted automatically to Google Cloud as part of terraform apply." + type = bool + default = false +} + +variable "allow_automatic_updates" { + description = <<-EOT + If false, disables automatic system package updates on the created instances. This feature is + only available on supported images (or images derived from them). For more details, see + https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates + EOT + type = bool + default = true + nullable = false +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/versions.tf new file mode 100644 index 0000000000..a1161e1354 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/versions.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + local = { + source = "hashicorp/local" + version = ">= 2.0.0" + } + random = { + source = "hashicorp/random" + version = ">= 3.0" + } + google = { + source = "hashicorp/google" + version = ">= 4.0" + } + } + required_version = ">= 1.1" +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/README.md b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/README.md new file mode 100644 index 0000000000..c20ca7dbeb --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/README.md @@ -0,0 +1,127 @@ +# Description + +This module creates a VM that acts as a login node to test and submit Google +Cloud Batch jobs. It is intended to be used along with the `batch-job-template` +module. + +This login node: + +- Uses the same VM settings as the first provided `batch-job-template`, such as + image, machine type, etc... +- Runs the same `startup-script` as the first provided `batch-job-template`. +- Has the same mounted file systems as the provided `batch-job-template`. +- Contains a folder with job templates generated by `batch-job-template` modules. + +Since the login node has the same mounted storage and is a homogeneous machine +to the Google Cloud Batch compute VMs, it can be used to inspect shared file +systems and test installed software before submitting a Google Cloud Batch job. + +## Example + +```yaml +- id: batch-job + source: modules/scheduler/batch-job-template + ... + +- id: batch-login + source: modules/scheduler/batch-login-node + use: [batch-job] + outputs: [instructions] +``` + +## Authentication + +To submit jobs from the login node, the service account attached to the VM needs +the `Batch Job Administrator` role. In most cases this service account will be +the Compute Engine default service account and will not be granted this role by +default. + +You can grant this role either by adding the `Batch Job Administrator` role to +the service account in the IAM page in the Google Cloud Console, or by running +the following command line: + +```bash +gcloud projects add-iam-policy-binding \ + --member=serviceAccount: \ + --role=roles/batch.jobsAdmin +``` + +## gcloud Batch Access + +Until the Google Cloud Batch API is generally available (GA), it may not be +available in all versions of the `gcloud` cli. You can test if the Google Cloud +Batch commands are available by running `gcloud [alpha|beta|] batch -h`. If the +Google Cloud Batch cli is not available it can generally be mitigated by either +updating `gcloud` by running `gcloud components update`, or using an image that +contains a more recent version of `gcloud`. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.14.0 | +| [google](#requirement\_google) | >= 3.83 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [login\_startup\_script](#module\_login\_startup\_script) | ../../scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_compute_instance_from_template.batch_login](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_from_template) | resource | +| [google_compute_instance_template.batch_instance_template](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance_template) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [batch\_job\_directory](#input\_batch\_job\_directory) | The path of the directory on the login node in which to place the Google Cloud Batch job template | `string` | `"/home/batch-jobs"` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the deployment, also used for the job\_id | `string` | n/a | yes | +| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | +| [gcloud\_version](#input\_gcloud\_version) | The version of the gcloud cli being used. Used for output instructions.
Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default
version). Typically supplied by a batch-job-template module. If multiple
batch-job-template modules supply the gcloud\_version, only the first will be used. | `string` | `""` | no | +| [instance\_template](#input\_instance\_template) | Login VM instance template self-link. Typically supplied by a
batch-job-template module. If multiple batch-job-template modules supply the
instance\_template, the first will be used. | `string` | n/a | yes | +| [job\_data](#input\_job\_data) | List of jobs and supporting data for each, typically provided via "use" from the batch-job-template module. | 
list(object({
    template_contents = string,
    filename          = string,
    id                = string
  }))
| n/a | yes | +| [job\_filename](#input\_job\_filename) | Deprecated (use `job_data`): The filename of the generated job template file. Typically supplied by a batch-job-template module. | `string` | `null` | no | +| [job\_id](#input\_job\_id) | Deprecated (use `job_data`): The ID for the Google Cloud Batch job. Typically supplied by a batch-job-template module for use in the output instructions. | `string` | `null` | no | +| [job\_template\_contents](#input\_job\_template\_contents) | Deprecated (use `job_data`): The contents of the Google Cloud Batch job template. Typically supplied by a batch-job-template module. | `string` | `null` | no | +| [labels](#input\_labels) | Labels to add to the login node. Key-value pairs | `map(string)` | n/a | yes | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. Typically supplied by a batch-job-template module. | 
list(object({
    server_ip             = string
    remote_mount          = string
    local_mount           = string
    fs_type               = string
    mount_options         = string
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The region in which to create the login node | `string` | n/a | yes | +| [startup\_script](#input\_startup\_script) | Startup script run before Google Cloud Batch job starts. Typically supplied by a batch-job-template module. | `string` | `null` | no | +| [zone](#input\_zone) | The zone in which to create the login node | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [instructions](#output\_instructions) | Instructions for accessing the login node and submitting Google Cloud Batch jobs | +| [login\_node\_name](#output\_login\_node\_name) | Name of the created VM | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/main.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/main.tf new file mode 100644 index 0000000000..6f539af122 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/main.tf @@ -0,0 +1,127 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "batch-login-node", ghpc_role = "scheduler" }) +} + +data "google_compute_instance_template" "batch_instance_template" { + name = var.instance_template +} + +locals { + job_template_runners = [for job in var.job_data : { + content = job.template_contents + destination = "${var.batch_job_directory}/${job.filename}" + type = "data" + }] + + instance_template_metadata = data.google_compute_instance_template.batch_instance_template.metadata + startup_metadata = { startup-script = module.login_startup_script.startup_script } + + oslogin_api_values = { + "DISABLE" = "FALSE" + "ENABLE" = "TRUE" + } + oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } + + login_metadata = merge(local.instance_template_metadata, local.startup_metadata, local.oslogin_metadata) + + batch_command_instructions = join("\n", [for job in var.job_data : <<-EOT + ## For job: ${job.id} ## + + Submit your job from login node: + gcloud ${var.gcloud_version} batch jobs submit ${job.id} --config=${var.batch_job_directory}/${job.filename} --location=${var.region} --project=${var.project_id} + + Check status: + gcloud ${var.gcloud_version} batch jobs describe ${job.id} --location=${var.region} --project=${var.project_id} | grep state: + + Delete job: + gcloud ${var.gcloud_version} batch jobs delete ${job.id} --location=${var.region} --project=${var.project_id} + + EOT + ]) + + list_all_jobs = <<-EOT + List all jobs: + gcloud ${var.gcloud_version} batch jobs list --project=${var.project_id} + EOT + + readme_contents = <<-EOT + # Batch Job Templates + + This folder contains Batch job templates created by the Cluster Toolkit. + These templates can be edited before submitting to Batch to capture more + complex workloads. + + Use the following commands to: + ${local.list_all_jobs} + + ${local.batch_command_instructions} + EOT + + # Construct startup script for network storage + storage_client_install_runners = [ + for i, ns in var.network_storage : merge(ns.client_install_runner, { + destination = "${i}-${ns.client_install_runner.destination}" + }) if ns.client_install_runner != null + ] + mount_runners = [ + for i, ns in var.network_storage : merge(ns.mount_runner, { + destination = "${i}-${ns.mount_runner.destination}" + }) if ns.mount_runner != null + ] + + startup_script_runner = { + content = var.startup_script != null ? var.startup_script : "echo 'Batch job template had no startup script'" + destination = "passed_startup_script.sh" + type = "shell" + } +} + +module "login_startup_script" { + source = "../../scripts/startup-script" + labels = local.labels + project_id = var.project_id + deployment_name = var.deployment_name + region = var.region + runners = concat( + local.storage_client_install_runners, + local.mount_runners, + [local.startup_script_runner], + local.job_template_runners, + [ + { + content = local.readme_contents + destination = "${var.batch_job_directory}/README.md" + type = "data" + } + ] + ) +} + +resource "google_compute_instance_from_template" "batch_login" { + name = "${var.deployment_name}-batch-login" + source_instance_template = var.instance_template + project = var.project_id + zone = var.zone + metadata = local.login_metadata + + service_account { + scopes = ["https://www.googleapis.com/auth/cloud-platform"] + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml new file mode 100644 index 0000000000..9af2319b4a --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml @@ -0,0 +1,21 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - batch.googleapis.com + - compute.googleapis.com + - storage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/outputs.tf new file mode 100644 index 0000000000..ea8eccf8d5 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/outputs.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "login_node_name" { + description = "Name of the created VM" + value = google_compute_instance_from_template.batch_login.name +} + +output "instructions" { + description = "Instructions for accessing the login node and submitting Google Cloud Batch jobs" + value = <<-EOT + + Batch job template files will be placed on the Batch login node in the following directory: + ${var.batch_job_directory} + + Use the following commands to: + SSH into the login node: + gcloud compute ssh --zone ${google_compute_instance_from_template.batch_login.zone} ${google_compute_instance_from_template.batch_login.name} --project ${google_compute_instance_from_template.batch_login.project} + + ${local.list_all_jobs} + + ${local.batch_command_instructions} + EOT +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/variables.tf new file mode 100644 index 0000000000..3b9caa7001 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/variables.tf @@ -0,0 +1,151 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "deployment_name" { + description = "Name of the deployment, also used for the job_id" + type = string +} + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "region" { + description = "The region in which to create the login node" + type = string +} + +variable "zone" { + description = "The zone in which to create the login node" + type = string +} + +variable "labels" { + description = "Labels to add to the login node. Key-value pairs" + type = map(string) +} + +variable "instance_template" { + description = <<-EOT + Login VM instance template self-link. Typically supplied by a + batch-job-template module. If multiple batch-job-template modules supply the + instance_template, the first will be used. + EOT + type = string +} + +variable "network_storage" { + description = "An array of network attached storage mounts to be configured. Typically supplied by a batch-job-template module." + type = list(object({ + server_ip = string + remote_mount = string + local_mount = string + fs_type = string + mount_options = string + client_install_runner = map(string) + mount_runner = map(string) + })) + default = [] +} + +variable "startup_script" { + description = "Startup script run before Google Cloud Batch job starts. Typically supplied by a batch-job-template module." + type = string + default = null +} + +variable "job_data" { + description = "List of jobs and supporting data for each, typically provided via \"use\" from the batch-job-template module." + type = list(object({ + template_contents = string, + filename = string, + id = string + })) + validation { + condition = length(distinct([for job in var.job_data : job.filename])) == length(var.job_data) + error_message = "All filenames in var.job_data must be unique." + } + validation { + condition = length(distinct([for job in var.job_data : job.id])) == length(var.job_data) + error_message = "All job IDs in var.job_data must be unique." + } +} + +# tflint-ignore: terraform_unused_declarations +variable "job_template_contents" { + description = "Deprecated (use `job_data`): The contents of the Google Cloud Batch job template. Typically supplied by a batch-job-template module." + type = string + default = null + validation { + condition = var.job_template_contents == null + error_message = "job_template_contents is deprecated. Please use `job_data` instead." + } +} + +# tflint-ignore: terraform_unused_declarations +variable "job_filename" { + description = "Deprecated (use `job_data`): The filename of the generated job template file. Typically supplied by a batch-job-template module." + type = string + default = null + validation { + condition = var.job_filename == null + error_message = "job_filename is deprecated. Please use `job_data` instead." + } +} + +# tflint-ignore: terraform_unused_declarations +variable "job_id" { + description = "Deprecated (use `job_data`): The ID for the Google Cloud Batch job. Typically supplied by a batch-job-template module for use in the output instructions." + type = string + default = null + validation { + condition = var.job_id == null + error_message = "job_id is deprecated. Please use `job_data` instead." + } +} + +variable "gcloud_version" { + description = <<-EOT + The version of the gcloud cli being used. Used for output instructions. + Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default + version). Typically supplied by a batch-job-template module. If multiple + batch-job-template modules supply the gcloud_version, only the first will be used. + EOT + type = string + default = "" + + validation { + condition = contains(["alpha", "beta", ""], var.gcloud_version) + error_message = "Allowed values for gcloud_version are 'alpha', 'beta', or '' (empty string)." + } +} + +variable "batch_job_directory" { + description = "The path of the directory on the login node in which to place the Google Cloud Batch job template" + type = string + default = "/home/batch-jobs" +} + +variable "enable_oslogin" { + description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." + type = string + default = "ENABLE" + validation { + condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) + error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." + } +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/versions.tf new file mode 100644 index 0000000000..5109db2aee --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.83" + } + } + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:batch-login-node/v1.57.0" + } + + required_version = ">= 0.14.0" +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/README.md b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/README.md new file mode 100644 index 0000000000..cdf6a9a3cf --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/README.md @@ -0,0 +1,215 @@ +## Description + +This module creates a Google Kubernetes Engine +([GKE](https://cloud.google.com/kubernetes-engine)) cluster. + +> **_NOTE:_** This is an experimental module and the functionality and +> documentation will likely be updated in the near future. This module has only +> been tested in limited capacity. + +### Example + +The following example creates a GKE cluster and a VPC designed to work with GKE. +See [VPC Network](#vpc-network) section for more information about network +requirements. + +```yaml + - id: network1 + source: modules/network/vpc + settings: + subnetwork_name: gke-subnet + secondary_ranges: + gke-subnet: + - range_name: pods + ip_cidr_range: 10.4.0.0/14 + - range_name: services + ip_cidr_range: 10.0.32.0/20 + + - id: gke_cluster + source: modules/scheduler/gke-cluster + use: [network1] +``` + +Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). + +### VPC Network + +This module is configured to create a +[VPC-native cluster](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips). +This means that alias IPs are used and that the subnetwork requires secondary +ranges for pods and services. In the example shown above these secondary ranges +are created in the VPC module. By default the `gke-cluster` module will look for +ranges with the names `pods` and `services`. These names can be configured using +the `pods_ip_range_name` and `services_ip_range_name` settings. + +### Multi-networking + +To [enable Multi-networking](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment), pass multivpc module to gke-cluster module as described in example below. Passing a multivpc module enables multi networking and [Dataplane V2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2?hl=en) on the cluster. + +```yaml + - id: network + source: modules/network/vpc + settings: + subnetwork_name: gke-subnet + secondary_ranges: + gke-subnet: + - range_name: pods + ip_cidr_range: 10.4.0.0/14 + - range_name: services + ip_cidr_range: 10.0.32.0/20 + + - id: multinetwork + source: modules/network/multivpc + settings: + network_name_prefix: multivpc-net + network_count: 8 + global_ip_address_range: 172.16.0.0/12 + subnetwork_cidr_suffix: 16 + + - id: gke-cluster + source: modules/scheduler/gke-cluster + use: [network, multinetwork] ## enables multi networking and Dataplane V2 on cluster + settings: + cluster_name: $(vars.deployment_name) +``` + +Find an example of multi networking in GKE [here](../../../examples/gke-a3-megagpu.yaml). + +### Cluster Limitations + +The current implementations has the following limitations: + +- Autopilot is disabled +- Auto-provisioning of new node pools is disabled +- Network policies are not supported +- General addon configuration is not supported +- Only regional cluster is supported + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 6.16 | +| [google-beta](#requirement\_google-beta) | >= 6.16 | +| [kubernetes](#requirement\_kubernetes) | >= 2.36 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 6.16 | +| [google-beta](#provider\_google-beta) | >= 6.16 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | +| [workload\_identity](#module\_workload\_identity) | terraform-google-modules/kubernetes-engine/google//modules/workload-identity | ~> 34.0 | + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_cluster) | resource | +| [google-beta_google_container_node_pool.system_node_pools](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_node_pool) | resource | +| [google-beta_google_container_engine_versions.version_prefix_filter](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/data-sources/google_container_engine_versions) | data source | +| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | +| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks enables multi networking and creates relevat network objects on the cluster. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [authenticator\_security\_group](#input\_authenticator\_security\_group) | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| [autoscaling\_profile](#input\_autoscaling\_profile) | (Beta) Optimize for utilization or availability when deciding to remove nodes. Can be BALANCED or OPTIMIZE\_UTILIZATION. | `string` | `"OPTIMIZE_UTILIZATION"` | no | +| [cloud\_dns\_config](#input\_cloud\_dns\_config) | Configuration for Using Cloud DNS for GKE.

additive\_vpc\_scope\_dns\_domain: This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = "CLOUD\_DNS" and cluster\_dns\_scope = "CLUSTER\_SCOPE" must both be set as well.
cluster\_dns: Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS.
cluster\_dns\_scope: The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE.
cluster\_dns\_domain: The suffix used for all cluster service records. | 
object({
    additive_vpc_scope_dns_domain = optional(string)
    cluster_dns                   = optional(string, "PROVIDER_UNSPECIFIED")
    cluster_dns_scope             = optional(string, "DNS_SCOPE_UNSPECIFIED")
    cluster_dns_domain            = optional(string)
  })
| 
{
  "additive_vpc_scope_dns_domain": null,
  "cluster_dns": "PROVIDER_UNSPECIFIED",
  "cluster_dns_domain": null,
  "cluster_dns_scope": "DNS_SCOPE_UNSPECIFIED"
}
| no | +| [cluster\_availability\_type](#input\_cluster\_availability\_type) | Type of cluster availability. Possible values are: {REGIONAL, ZONAL} | `string` | `"REGIONAL"` | no | +| [cluster\_reference\_type](#input\_cluster\_reference\_type) | How the google\_container\_node\_pool.system\_node\_pools refers to the cluster. Possible values are: {SELF\_LINK, NAME} | `string` | `"SELF_LINK"` | no | +| [configure\_workload\_identity\_sa](#input\_configure\_workload\_identity\_sa) | When true, a kubernetes service account will be created and bound using workload identity to the service account used to create the cluster. | `bool` | `false` | no | +| [default\_max\_pods\_per\_node](#input\_default\_max\_pods\_per\_node) | The default maximum number of pods per node in this cluster. | `number` | `null` | no | +| [deletion\_protection](#input\_deletion\_protection) | "Determines if the cluster can be deleted by gcluster commands or not".
To delete a cluster provisioned with deletion\_protection set to true, you must first set it to false and apply the changes.
Then proceed with deletion as usual. | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment. Used in the GKE cluster name by default and can be configured with `prefix_with_deployment_name`. | `string` | n/a | yes | +| [enable\_dataplane\_v2](#input\_enable\_dataplane\_v2) | Enables [Dataplane v2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2). This setting is immutable on clusters. If null, will default to false unless using multi-networking, in which case it will default to true | `bool` | `null` | no | +| [enable\_dcgm\_monitoring](#input\_enable\_dcgm\_monitoring) | Enable GKE to collect DCGM metrics | `bool` | `false` | no | +| [enable\_external\_dns\_endpoint](#input\_enable\_external\_dns\_endpoint) | Allow [DNS-based approach](https://cloud.google.com/kubernetes-engine/docs/concepts/network-isolation#dns-based_endpoint) for accessing the GKE control plane.
Refer this [dedicated blog](https://cloud.google.com/blog/products/containers-kubernetes/new-dns-based-endpoint-for-the-gke-control-plane) for more details. | `bool` | `false` | no | +| [enable\_filestore\_csi](#input\_enable\_filestore\_csi) | The status of the Filestore Container Storage Interface (CSI) driver addon, which allows the usage of filestore instance as volumes. | `bool` | `false` | no | +| [enable\_gcsfuse\_csi](#input\_enable\_gcsfuse\_csi) | The status of the GCSFuse Container Storage Interface (CSI) driver addon, which allows the usage of a GCS bucket as volumes. | `bool` | `false` | no | +| [enable\_k8s\_beta\_apis](#input\_enable\_k8s\_beta\_apis) | List of Enabled Kubernetes Beta APIs. | `list(string)` | `null` | no | +| [enable\_master\_global\_access](#input\_enable\_master\_global\_access) | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `false` | no | +| [enable\_multi\_networking](#input\_enable\_multi\_networking) | Enables [multi networking](https://cloud.google.com/kubernetes-engine/docs/how-to/setup-multinetwork-support-for-pods#create-a-gke-cluster) (Requires GKE Enterprise). This setting is immutable on clusters and enables [Dataplane V2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2?hl=en). If null, will determine state based on if additional\_networks are passed in. | `bool` | `null` | no | +| [enable\_node\_local\_dns\_cache](#input\_enable\_node\_local\_dns\_cache) | Enable GKE NodeLocal DNSCache addon to improve DNS lookup latency | `bool` | `false` | no | +| [enable\_parallelstore\_csi](#input\_enable\_parallelstore\_csi) | The status of the Google Compute Engine Parallelstore Container Storage Interface (CSI) driver addon, which allows the usage of a parallelstore as volumes. | `bool` | `false` | no | +| [enable\_persistent\_disk\_csi](#input\_enable\_persistent\_disk\_csi) | The status of the Google Compute Engine Persistent Disk Container Storage Interface (CSI) driver addon, which allows the usage of a PD as volumes. | `bool` | `true` | no | +| [enable\_private\_endpoint](#input\_enable\_private\_endpoint) | (Beta) Whether the master's internal IP address is used as the cluster endpoint. | `bool` | `true` | no | +| [enable\_private\_ipv6\_google\_access](#input\_enable\_private\_ipv6\_google\_access) | The private IPv6 google access type for the VMs in this subnet. | `bool` | `true` | no | +| [enable\_private\_nodes](#input\_enable\_private\_nodes) | (Beta) Whether nodes have internal IP addresses only. | `bool` | `true` | no | +| [enable\_ray\_operator](#input\_enable\_ray\_operator) | The status of the Ray operator addon, This feature enables Kubernetes APIs for managing and scaling Ray clusters and jobs. You control and are responsible for managing ray.io custom resources in your cluster. This feature is not compatible with GKE clusters that already have another Ray operator installed. Supports clusters on Kubernetes version 1.29.8-gke.1054000 or later. | `bool` | `false` | no | +| [gcp\_public\_cidrs\_access\_enabled](#input\_gcp\_public\_cidrs\_access\_enabled) | Whether the cluster master is accessible via all the Google Compute Engine Public IPs. To view this list of IP addresses look here https://cloud.google.com/compute/docs/faq#find_ip_range | `bool` | `false` | no | +| [k8s\_network\_names](#input\_k8s\_network\_names) | Kubernetes network names details for GKE. If starting index is not specified for gvnic or rdma, it would be set to the default values. | 
object({
    gvnic_prefix      = optional(string, "")
    gvnic_start_index = optional(number, 1)
    gvnic_postfix     = optional(string, "")
    rdma_prefix       = optional(string, "")
    rdma_start_index  = optional(number, 0)
    rdma_postfix      = optional(string, "")
  })
| 
{
  "gvnic_postfix": "",
  "gvnic_prefix": "gvnic-",
  "gvnic_start_index": 1,
  "rdma_postfix": "",
  "rdma_prefix": "rdma-",
  "rdma_start_index": 0
}
| no | +| [k8s\_service\_account\_name](#input\_k8s\_service\_account\_name) | Kubernetes service account name to use with the gke cluster | `string` | `"workload-identity-k8s-sa"` | no | +| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | +| [maintenance\_exclusions](#input\_maintenance\_exclusions) | List of maintenance exclusions. A cluster can have up to three. | 
list(object({
    name            = string
    start_time      = string
    end_time        = string
    exclusion_scope = string
  }))
| `[]` | no | +| [maintenance\_start\_time](#input\_maintenance\_start\_time) | Start time for daily maintenance operations. Specified in GMT with `HH:MM` format. | `string` | `"09:00"` | no | +| [master\_authorized\_networks](#input\_master\_authorized\_networks) | External network that can access Kubernetes master through HTTPS. Must be specified in CIDR notation. | 
list(object({
    cidr_block   = string
    display_name = string
  }))
| `[]` | no | +| [master\_ipv4\_cidr\_block](#input\_master\_ipv4\_cidr\_block) | (Beta) The IP range in CIDR notation to use for the hosted master network. | `string` | `"172.16.0.32/28"` | no | +| [min\_master\_version](#input\_min\_master\_version) | The minimum version of the master. If unset, the cluster's version will be set by GKE to the version of the most recent official release. | `string` | `null` | no | +| [name\_suffix](#input\_name\_suffix) | Custom cluster name postpended to the `deployment_name`. See `prefix_with_deployment_name`. | `string` | `""` | no | +| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to host the cluster given in the format: `projects//global/networks/`. | `string` | n/a | yes | +| [networking\_mode](#input\_networking\_mode) | Determines whether alias IPs or routes will be used for pod IPs in the cluster. Options are VPC\_NATIVE or ROUTES. VPC\_NATIVE enables IP aliasing. The default is VPC\_NATIVE. | `string` | `"VPC_NATIVE"` | no | +| [pods\_ip\_range\_name](#input\_pods\_ip\_range\_name) | The name of the secondary subnet ip range to use for pods. | `string` | `"pods"` | no | +| [prefix\_with\_deployment\_name](#input\_prefix\_with\_deployment\_name) | If true, cluster name will be prefixed by `deployment_name` (ex: -). | `bool` | `true` | no | +| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | +| [region](#input\_region) | The region to host the cluster in. | `string` | n/a | yes | +| [release\_channel](#input\_release\_channel) | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. | `string` | `"UNSPECIFIED"` | no | +| [service\_account](#input\_service\_account) | DEPRECATED: use service\_account\_email and scopes. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | +| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the system node pool | `string` | `null` | no | +| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the system node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | +| [services\_ip\_range\_name](#input\_services\_ip\_range\_name) | The name of the secondary subnet range to use for services. | `string` | `"services"` | no | +| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to host the cluster in. | `string` | n/a | yes | +| [system\_node\_pool\_disk\_size\_gb](#input\_system\_node\_pool\_disk\_size\_gb) | Size of disk for each node of the system node pool. | `number` | `100` | no | +| [system\_node\_pool\_disk\_type](#input\_system\_node\_pool\_disk\_type) | Disk type for each node of the system node pool. | `string` | `null` | no | +| [system\_node\_pool\_enable\_secure\_boot](#input\_system\_node\_pool\_enable\_secure\_boot) | Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info. | `bool` | `true` | no | +| [system\_node\_pool\_enabled](#input\_system\_node\_pool\_enabled) | Create a system node pool. | `bool` | `true` | no | +| [system\_node\_pool\_image\_type](#input\_system\_node\_pool\_image\_type) | The default image type used by NAP once a new node pool is being created. Use either COS\_CONTAINERD or UBUNTU\_CONTAINERD. | `string` | `"COS_CONTAINERD"` | no | +| [system\_node\_pool\_kubernetes\_labels](#input\_system\_node\_pool\_kubernetes\_labels) | Kubernetes labels to be applied to each node in the node group. Key-value pairs.
(The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) | `map(string)` | `null` | no | +| [system\_node\_pool\_machine\_type](#input\_system\_node\_pool\_machine\_type) | Machine type for the system node pool. | `string` | `"e2-standard-4"` | no | +| [system\_node\_pool\_name](#input\_system\_node\_pool\_name) | Name of the system node pool. | `string` | `"system"` | no | +| [system\_node\_pool\_node\_count](#input\_system\_node\_pool\_node\_count) | The total min and max nodes to be maintained in the system node pool. | 
object({
    total_min_nodes = number
    total_max_nodes = number
  })
| 
{
  "total_max_nodes": 10,
  "total_min_nodes": 2
}
| no | +| [system\_node\_pool\_taints](#input\_system\_node\_pool\_taints) | Taints to be applied to the system node pool. | 
list(object({
    key    = string
    value  = any
    effect = string
  }))
| 
[
  {
    "effect": "NO_SCHEDULE",
    "key": "components.gke.io/gke-managed-components",
    "value": true
  }
]
| no | +| [system\_node\_pool\_zones](#input\_system\_node\_pool\_zones) | The zones to use for the system node pool. If not specified, the cluster default node zone(s) will be used. | `list(string)` | `null` | no | +| [timeout\_create](#input\_timeout\_create) | Timeout for creating a node pool | `string` | `null` | no | +| [timeout\_update](#input\_timeout\_update) | Timeout for updating a node pool | `string` | `null` | no | +| [upgrade\_settings](#input\_upgrade\_settings) | Defines gke cluster upgrade settings. It is highly recommended that you define all max\_surge and max\_unavailable.
If max\_surge is not specified, it would be set to a default value of 0.
If max\_unavailable is not specified, it would be set to a default value of 1. | 
object({
    strategy        = string
    max_surge       = optional(number)
    max_unavailable = optional(number)
  })
| 
{
  "max_surge": 0,
  "max_unavailable": 1,
  "strategy": "SURGE"
}
| no | +| [version\_prefix](#input\_version\_prefix) | If provided, Terraform will only return versions that match the string prefix. For example, `1.31.` will match all `1.31` series releases. Since this is just a string match, it's recommended that you append a `.` after minor versions to ensure that prefixes such as `1.3` don't match versions like `1.30.1-gke.10` accidentally. | `string` | `"1.31."` | no | +| [zone](#input\_zone) | Zone for a zonal cluster. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_id](#output\_cluster\_id) | An identifier for the resource with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | +| [gke\_cluster\_exists](#output\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | +| [gke\_version](#output\_gke\_version) | GKE cluster's version. | +| [instructions](#output\_instructions) | Instructions on how to connect to the created cluster. | +| [k8s\_service\_account\_name](#output\_k8s\_service\_account\_name) | Name of k8s service account. | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/main.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/main.tf new file mode 100644 index 0000000000..0ec1fde35b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/main.tf @@ -0,0 +1,447 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "gke-cluster", ghpc_role = "scheduler" }) +} + +locals { + upgrade_settings = { + strategy = var.upgrade_settings.strategy + max_surge = coalesce(var.upgrade_settings.max_surge, 0) + max_unavailable = coalesce(var.upgrade_settings.max_unavailable, 1) + } +} + +locals { + dash = var.prefix_with_deployment_name && var.name_suffix != "" ? "-" : "" + prefix = var.prefix_with_deployment_name ? var.deployment_name : "" + name_maybe_empty = "${local.prefix}${local.dash}${var.name_suffix}" + name = local.name_maybe_empty != "" ? local.name_maybe_empty : "NO-NAME-GIVEN" + + cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ + security_group = var.authenticator_security_group + }] + + default_sa_email = "${data.google_project.project.number}-compute@developer.gserviceaccount.com" + sa_email = coalesce(var.service_account_email, local.default_sa_email) + + # additional VPCs enable multi networking + derived_enable_multi_networking = coalesce(var.enable_multi_networking, length(var.additional_networks) > 0) + + # multi networking needs enabled Dataplane v2 + derived_enable_dataplane_v2 = coalesce(var.enable_dataplane_v2, local.derived_enable_multi_networking) + + default_monitoring_component = [ + "SYSTEM_COMPONENTS", + "POD", + "DAEMONSET", + "DEPLOYMENT", + "STATEFULSET", + "STORAGE", + "HPA", + "CADVISOR", + "KUBELET" + ] + + default_logging_component = [ + "SYSTEM_COMPONENTS", + "WORKLOADS" + ] +} + +data "google_project" "project" { + project_id = var.project_id +} + +data "google_container_engine_versions" "version_prefix_filter" { + provider = google-beta + location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region + version_prefix = var.version_prefix +} + +locals { + master_version = var.min_master_version != null ? var.min_master_version : data.google_container_engine_versions.version_prefix_filter.latest_master_version +} + +resource "google_container_cluster" "gke_cluster" { + provider = google-beta + + project = var.project_id + name = local.name + location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region + resource_labels = local.labels + networking_mode = var.networking_mode + # decouple node pool lifecycle from cluster life cycle + remove_default_node_pool = true + initial_node_count = 1 # must be set when remove_default_node_pool is set + node_locations = var.system_node_pool_zones + + deletion_protection = var.deletion_protection + + dynamic "enable_k8s_beta_apis" { + for_each = var.enable_k8s_beta_apis != null ? [1] : [] + content { + enabled_apis = var.enable_k8s_beta_apis + } + } + + network = var.network_id + subnetwork = var.subnetwork_self_link + + # Note: the existence of the "master_authorized_networks_config" block enables + # the master authorized networks even if it's empty. + master_authorized_networks_config { + dynamic "cidr_blocks" { + for_each = var.master_authorized_networks + content { + cidr_block = cidr_blocks.value.cidr_block + display_name = cidr_blocks.value.display_name + } + } + gcp_public_cidrs_access_enabled = var.gcp_public_cidrs_access_enabled + } + + private_ipv6_google_access = var.enable_private_ipv6_google_access ? "PRIVATE_IPV6_GOOGLE_ACCESS_TO_GOOGLE" : null + default_max_pods_per_node = var.default_max_pods_per_node + master_auth { + client_certificate_config { + issue_client_certificate = false + } + } + + enable_shielded_nodes = true + + cluster_autoscaling { + # Controls auto provisioning of node-pools + enabled = false + + # Controls autoscaling algorithm of node-pools + autoscaling_profile = var.autoscaling_profile + } + + datapath_provider = local.derived_enable_dataplane_v2 ? "ADVANCED_DATAPATH" : "LEGACY_DATAPATH" + + enable_multi_networking = local.derived_enable_multi_networking + + network_policy { + # Enabling NetworkPolicy for clusters with DatapathProvider=ADVANCED_DATAPATH + # is not allowed. Dataplane V2 will take care of network policy enforcement + # instead. + enabled = false + # GKE Dataplane V2 support. This must be set to PROVIDER_UNSPECIFIED in + # order to let the datapath_provider take effect. + # https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/656#issuecomment-720398658 + provider = "PROVIDER_UNSPECIFIED" + } + + private_cluster_config { + enable_private_nodes = var.enable_private_nodes + enable_private_endpoint = var.enable_private_endpoint + master_ipv4_cidr_block = var.master_ipv4_cidr_block + master_global_access_config { + enabled = var.enable_master_global_access + } + } + + ip_allocation_policy { + cluster_secondary_range_name = var.pods_ip_range_name + services_secondary_range_name = var.services_ip_range_name + } + + workload_identity_config { + workload_pool = "${var.project_id}.svc.id.goog" + } + + dynamic "authenticator_groups_config" { + for_each = local.cluster_authenticator_security_group + content { + security_group = authenticator_groups_config.value.security_group + } + } + + release_channel { + channel = var.release_channel + } + min_master_version = local.master_version + + maintenance_policy { + daily_maintenance_window { + start_time = var.maintenance_start_time + } + + dynamic "maintenance_exclusion" { + for_each = var.maintenance_exclusions + content { + exclusion_name = maintenance_exclusion.value.name + start_time = maintenance_exclusion.value.start_time + end_time = maintenance_exclusion.value.end_time + exclusion_options { + scope = maintenance_exclusion.value.exclusion_scope + } + } + } + } + + dynamic "dns_config" { + for_each = var.cloud_dns_config != null ? [1] : [] + content { + additive_vpc_scope_dns_domain = var.cloud_dns_config.additive_vpc_scope_dns_domain + cluster_dns = var.cloud_dns_config.cluster_dns + cluster_dns_scope = var.cloud_dns_config.cluster_dns_scope + cluster_dns_domain = var.cloud_dns_config.cluster_dns_domain + } + } + + addons_config { + gcp_filestore_csi_driver_config { + enabled = var.enable_filestore_csi + } + gcs_fuse_csi_driver_config { + enabled = var.enable_gcsfuse_csi + } + gce_persistent_disk_csi_driver_config { + enabled = var.enable_persistent_disk_csi + } + dns_cache_config { + enabled = var.enable_node_local_dns_cache + } + parallelstore_csi_driver_config { + enabled = var.enable_parallelstore_csi + } + ray_operator_config { + enabled = var.enable_ray_operator + } + } + + timeouts { + create = var.timeout_create + update = var.timeout_update + } + + node_config { + shielded_instance_config { + enable_secure_boot = var.system_node_pool_enable_secure_boot + enable_integrity_monitoring = true + } + } + + control_plane_endpoints_config { + dns_endpoint_config { + allow_external_traffic = var.enable_external_dns_endpoint + } + } + + lifecycle { + # Ignore all changes to the default node pool. It's being removed after creation. + ignore_changes = [ + node_config, + min_master_version, + ] + precondition { + condition = var.default_max_pods_per_node == null || var.networking_mode == "VPC_NATIVE" + error_message = "default_max_pods_per_node does not work on `routes-based` clusters, that don't have IP Aliasing enabled." + } + precondition { + condition = coalesce(var.enable_dataplane_v2, true) || !local.derived_enable_multi_networking + error_message = "'enable_dataplane_v2' cannot be false when enabling multi networking." + } + precondition { + condition = coalesce(var.enable_multi_networking, true) || length(var.additional_networks) == 0 + error_message = "'enable_multi_networking' cannot be false when using multivpc module, which passes additional_networks." + } + } + + monitoring_config { + enable_components = var.enable_dcgm_monitoring ? concat(local.default_monitoring_component, ["DCGM"]) : local.default_monitoring_component + managed_prometheus { + enabled = true + } + } + + logging_config { + enable_components = local.default_logging_component + } +} + +# We define explicit node pools, so that it can be modified without +# having to destroy the entire cluster. +resource "google_container_node_pool" "system_node_pools" { + provider = google-beta + count = var.system_node_pool_enabled ? 1 : 0 + + project = var.project_id + name = var.system_node_pool_name + cluster = var.cluster_reference_type == "NAME" ? google_container_cluster.gke_cluster.name : google_container_cluster.gke_cluster.self_link + location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region + node_locations = var.system_node_pool_zones + version = local.master_version + + autoscaling { + total_min_node_count = var.system_node_pool_node_count.total_min_nodes + total_max_node_count = var.system_node_pool_node_count.total_max_nodes + } + + upgrade_settings { + strategy = local.upgrade_settings.strategy + max_surge = local.upgrade_settings.max_surge + max_unavailable = local.upgrade_settings.max_unavailable + } + + management { + auto_repair = true + auto_upgrade = true + } + + node_config { + labels = var.system_node_pool_kubernetes_labels + resource_labels = local.labels + service_account = var.service_account_email + oauth_scopes = var.service_account_scopes + machine_type = var.system_node_pool_machine_type + disk_size_gb = var.system_node_pool_disk_size_gb + disk_type = var.system_node_pool_disk_type + + dynamic "taint" { + for_each = var.system_node_pool_taints + content { + key = taint.value.key + value = taint.value.value + effect = taint.value.effect + } + } + + # Forcing the use of the Container-optimized image, as it is the only + # image with the proper logging daemon installed. + # + # cos images use Shielded VMs since v1.13.6-gke.0. + # https://cloud.google.com/kubernetes-engine/docs/how-to/node-images + # + # We use COS_CONTAINERD to be compatible with (optional) gVisor. + # https://cloud.google.com/kubernetes-engine/docs/how-to/sandbox-pods + image_type = var.system_node_pool_image_type + + shielded_instance_config { + enable_secure_boot = var.system_node_pool_enable_secure_boot + enable_integrity_monitoring = true + } + + gvnic { + enabled = var.system_node_pool_image_type == "COS_CONTAINERD" + } + + # Implied by Workload Identity + workload_metadata_config { + mode = "GKE_METADATA" + } + # Implied by workload identity. + metadata = { + "disable-legacy-endpoints" = "true" + } + } + + lifecycle { + ignore_changes = [ + node_config[0].labels, + node_config[0].taint, + version, + ] + precondition { + condition = contains(["SURGE"], local.upgrade_settings.strategy) + error_message = "Only SURGE strategy is supported" + } + precondition { + condition = local.upgrade_settings.max_unavailable >= 0 + error_message = "max_unavailable should be set to 0 or greater" + } + precondition { + condition = local.upgrade_settings.max_surge >= 0 + error_message = "max_surge should be set to 0 or greater" + } + precondition { + condition = local.upgrade_settings.max_unavailable > 0 || local.upgrade_settings.max_surge > 0 + error_message = "At least one of max_unavailable or max_surge must greater than 0" + } + } +} + +data "google_client_config" "default" {} + +provider "kubernetes" { + host = "https://${google_container_cluster.gke_cluster.endpoint}" + cluster_ca_certificate = base64decode(google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate) + token = data.google_client_config.default.access_token +} + +module "workload_identity" { + count = var.configure_workload_identity_sa ? 1 : 0 + source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity" + version = "~> 34.0" + + use_existing_gcp_sa = true + name = var.k8s_service_account_name + gcp_sa_name = local.sa_email + project_id = var.project_id + + # https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1059 + depends_on = [ + data.google_project.project, + google_container_cluster.gke_cluster + ] +} + +locals { + k8s_service_account_name = one(module.workload_identity[*].k8s_service_account_name) +} + +locals { + # Separate gvnic and rdma networks and assign indexes + gvnic_networks = [for idx, net in [for n in var.additional_networks : n if strcontains(upper(n.nic_type), "GVNIC")] : + merge(net, { name = "${var.k8s_network_names.gvnic_prefix}${idx + var.k8s_network_names.gvnic_start_index}${var.k8s_network_names.gvnic_postfix}" }) + ] + + rdma_networks = [for idx, net in [for n in var.additional_networks : n if strcontains(upper(n.nic_type), "RDMA")] : + merge(net, { name = "${var.k8s_network_names.rdma_prefix}${idx + var.k8s_network_names.rdma_start_index}${var.k8s_network_names.rdma_postfix}" }) + ] + + all_networks = concat(local.gvnic_networks, local.rdma_networks) +} + +module "kubectl_apply" { + source = "../../management/kubectl-apply" + + cluster_id = google_container_cluster.gke_cluster.id + project_id = var.project_id + + apply_manifests = flatten([ + for idx, network_info in local.all_networks : [ + { + source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", + template_vars = { + name = network_info.name, + network_name = network_info.network + subnetwork_name = network_info.subnetwork, + device_mode = strcontains(upper(network_info.nic_type), "RDMA") ? "RDMA" : "NetDevice" + } + }, + { + source = "${path.module}/templates/network-object.yaml.tftpl", + template_vars = { name = network_info.name } + } + ] + ]) +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml new file mode 100644 index 0000000000..bd1517ce8f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/outputs.tf new file mode 100644 index 0000000000..3326a5468e --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/outputs.tf @@ -0,0 +1,104 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cluster_id" { + description = "An identifier for the resource with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." + value = google_container_cluster.gke_cluster.id +} + +output "gke_cluster_exists" { + description = "A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations." + value = true + depends_on = [ + google_container_cluster.gke_cluster + ] +} + +locals { + private_endpoint_message = trimspace( + <<-EOT + This cluster was created with 'enable_private_endpoint: true'. + It cannot be accessed from a public IP addresses. + One way to access this cluster is from a VM created in the GKE cluster subnet. + EOT + ) + master_authorized_networks_message = length(var.master_authorized_networks) == 0 ? "" : trimspace( + <<-EOT + The following networks have been authorized to access this cluster: + ${join("\n", [for x in var.master_authorized_networks : " ${x.display_name}: ${x.cidr_block}"])}" + EOT + ) + public_endpoint_message = trimspace( + <<-EOT + To add authorized networks you can allowlist your IP with this command: + gcloud container clusters update ${google_container_cluster.gke_cluster.name} \ + --region ${google_container_cluster.gke_cluster.location} \ + --project ${var.project_id} \ + --enable-master-authorized-networks \ + --master-authorized-networks /32 + EOT + ) + allowlist_your_ip_message = var.enable_private_endpoint ? local.private_endpoint_message : local.public_endpoint_message + kubernetes_service_account_message = local.k8s_service_account_name == null ? "" : trimspace( + <<-EOT + Use the following Kubernetes Service Account in the default namespace to run your workloads: + ${local.k8s_service_account_name} + The GCP Service Account mapped to this Kubernetes Service Account is: + ${local.sa_email} + EOT + ) + kubernetes_cluster_fetch_credential_message = var.enable_external_dns_endpoint ? trimspace( + <<-EOT + Use the following command to fetch credentials for the created cluster: + gcloud container clusters get-credentials ${google_container_cluster.gke_cluster.name} \ + --region ${google_container_cluster.gke_cluster.location} \ + --project ${var.project_id} \ + --dns-endpoint + EOT + ) : trimspace( + <<-EOT + Use the following command to fetch credentials for the created cluster: + gcloud container clusters get-credentials ${google_container_cluster.gke_cluster.name} \ + --region ${google_container_cluster.gke_cluster.location} \ + --project ${var.project_id} + EOT + ) +} + +output "instructions" { + description = "Instructions on how to connect to the created cluster." + value = trimspace( + <<-EOT + ${local.master_authorized_networks_message} + + ${local.allowlist_your_ip_message} + + ${local.kubernetes_cluster_fetch_credential_message} + + ${local.kubernetes_service_account_message} + EOT + ) +} + +output "k8s_service_account_name" { + description = "Name of k8s service account." + value = local.k8s_service_account_name +} + +output "gke_version" { + description = "GKE cluster's version." + value = google_container_cluster.gke_cluster.master_version +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl new file mode 100644 index 0000000000..d376a1a760 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl @@ -0,0 +1,9 @@ +--- +apiVersion: networking.gke.io/v1 +kind: GKENetworkParamSet +metadata: + name: ${name} +spec: + vpc: ${network_name} + vpcSubnet: ${subnetwork_name} + deviceMode: ${device_mode} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl new file mode 100644 index 0000000000..1571a92692 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl @@ -0,0 +1,11 @@ +--- +apiVersion: networking.gke.io/v1 +kind: Network +metadata: + name: ${name} +spec: + parametersRef: + group: networking.gke.io + kind: GKENetworkParamSet + name: ${name} + type: Device diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/variables.tf new file mode 100644 index 0000000000..7dcb9d0ef0 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/variables.tf @@ -0,0 +1,521 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "The project ID to host the cluster in." + type = string +} + +variable "name_suffix" { + description = "Custom cluster name postpended to the `deployment_name`. See `prefix_with_deployment_name`." + type = string + default = "" +} + +variable "deployment_name" { + description = "Name of the HPC deployment. Used in the GKE cluster name by default and can be configured with `prefix_with_deployment_name`." + type = string +} + +variable "prefix_with_deployment_name" { + description = "If true, cluster name will be prefixed by `deployment_name` (ex: -)." + type = bool + default = true +} + +variable "region" { + description = "The region to host the cluster in." + type = string +} + +variable "zone" { + description = "Zone for a zonal cluster." + default = null + type = string +} + +variable "network_id" { + description = "The ID of the GCE VPC network to host the cluster given in the format: `projects//global/networks/`." + type = string + validation { + condition = length(split("/", var.network_id)) == 5 + error_message = "The network id must be provided in the following format: projects//global/networks/." + } +} + +variable "subnetwork_self_link" { + description = "The self link of the subnetwork to host the cluster in." + type = string +} + +variable "pods_ip_range_name" { + description = "The name of the secondary subnet ip range to use for pods." + type = string + default = "pods" +} + +variable "services_ip_range_name" { + description = "The name of the secondary subnet range to use for services." + type = string + default = "services" +} + +variable "enable_private_ipv6_google_access" { + description = "The private IPv6 google access type for the VMs in this subnet." + type = bool + default = true +} + +variable "release_channel" { + description = "The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`." + type = string + default = "UNSPECIFIED" +} + +variable "min_master_version" { + description = "The minimum version of the master. If unset, the cluster's version will be set by GKE to the version of the most recent official release." + type = string + default = null +} + +variable "version_prefix" { + description = "If provided, Terraform will only return versions that match the string prefix. For example, `1.31.` will match all `1.31` series releases. Since this is just a string match, it's recommended that you append a `.` after minor versions to ensure that prefixes such as `1.3` don't match versions like `1.30.1-gke.10` accidentally." + type = string + default = "1.31." +} + +variable "maintenance_start_time" { + description = "Start time for daily maintenance operations. Specified in GMT with `HH:MM` format." + type = string + default = "09:00" +} + +variable "maintenance_exclusions" { + description = "List of maintenance exclusions. A cluster can have up to three." + type = list(object({ + name = string + start_time = string + end_time = string + exclusion_scope = string + })) + default = [] + validation { + condition = alltrue([ + for x in var.maintenance_exclusions : + contains(["NO_UPGRADES", "NO_MINOR_UPGRADES", "NO_MINOR_OR_NODE_UPGRADES"], x.exclusion_scope) + ]) + error_message = "`exclusion_scope` must be set to `NO_UPGRADES` OR `NO_MINOR_UPGRADES` OR `NO_MINOR_OR_NODE_UPGRADES`." + } +} + +variable "cloud_dns_config" { + description = < **_NOTE:_** The `project_id` and `region` settings would be inferred from the +> deployment variables of the same name, but they are included here for clarity. + +### Multi-networking + +To create network objects in GKE cluster, you can pass a multivpc module to a pre-existing-gke-cluster module instead of [applying a manifest manually](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment). + +```yaml + - id: network + source: modules/network/vpc + + - id: multinetwork + source: modules/network/multivpc + settings: + network_name_prefix: multivpc-net + network_count: 8 + global_ip_address_range: 172.16.0.0/12 + subnetwork_cidr_suffix: 16 + + - id: existing-gke-cluster ## multinetworking must be enabled in advance when cluster creation + source: modules/scheduler/pre-existing-gke-cluster + use: [multinetwork] + settings: + cluster_name: $(vars.deployment_name) +``` + +## License + + +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | > 5.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | > 5.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_container_cluster.existing_gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks creates relevat network objects on the cluster. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [cluster\_name](#input\_cluster\_name) | Name of the existing cluster | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | Project that hosts the existing cluster | `string` | n/a | yes | +| [rdma\_subnetwork\_name\_prefix](#input\_rdma\_subnetwork\_name\_prefix) | Prefix of the RDMA subnetwork names | `string` | `null` | no | +| [region](#input\_region) | Region in which to search for the cluster | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_id](#output\_cluster\_id) | An identifier for the gke cluster with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | +| [gke\_cluster\_exists](#output\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster exists. | +| [gke\_version](#output\_gke\_version) | GKE cluster's version. | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf new file mode 100644 index 0000000000..926d2be100 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_container_cluster" "existing_gke_cluster" { + name = var.cluster_name + project = var.project_id + location = var.region +} + +locals { + rdma_networks = [for network_info in var.additional_networks : network_info if strcontains(upper(network_info.nic_type), "RDMA")] + non_rdma_networks = [for network_info in var.additional_networks : network_info if !strcontains(upper(network_info.nic_type), "RDMA")] + apply_manifests_rdma_networks = flatten([ + for idx, network_info in local.rdma_networks : [ + { + source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", + template_vars = { + name = "${var.rdma_subnetwork_name_prefix}-${idx}", + network_name = network_info.network + subnetwork_name = "${var.rdma_subnetwork_name_prefix}-${idx}", + device_mode = "RDMA" + } + }, + { + source = "${path.module}/templates/network-object.yaml.tftpl", + template_vars = { name = "${var.rdma_subnetwork_name_prefix}-${idx}" } + } + ] + ]) + + apply_manifests_non_rdma_networks = flatten([ + for idx, network_info in local.non_rdma_networks : [ + { + source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", + template_vars = { + name = network_info.subnetwork + network_name = network_info.network + subnetwork_name = network_info.subnetwork + device_mode = "NetDevice" + } + }, + { + source = "${path.module}/templates/network-object.yaml.tftpl", + template_vars = { name = network_info.subnetwork } + } + ] + ]) +} + +module "kubectl_apply" { + source = "../../management/kubectl-apply" + + cluster_id = data.google_container_cluster.existing_gke_cluster.id + project_id = var.project_id + + apply_manifests = concat(local.apply_manifests_non_rdma_networks, local.apply_manifests_rdma_networks) +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml new file mode 100644 index 0000000000..17bedb471b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - container.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf new file mode 100644 index 0000000000..8884ee30b0 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cluster_id" { + description = "An identifier for the gke cluster with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." + value = data.google_container_cluster.existing_gke_cluster.id +} + +output "gke_cluster_exists" { + description = "A static flag that signals to downstream modules that a cluster exists." + value = true + depends_on = [ + data.google_container_cluster.existing_gke_cluster + ] +} + +output "gke_version" { + description = "GKE cluster's version." + value = data.google_container_cluster.existing_gke_cluster.master_version +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl new file mode 100644 index 0000000000..d376a1a760 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl @@ -0,0 +1,9 @@ +--- +apiVersion: networking.gke.io/v1 +kind: GKENetworkParamSet +metadata: + name: ${name} +spec: + vpc: ${network_name} + vpcSubnet: ${subnetwork_name} + deviceMode: ${device_mode} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl new file mode 100644 index 0000000000..1571a92692 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl @@ -0,0 +1,11 @@ +--- +apiVersion: networking.gke.io/v1 +kind: Network +metadata: + name: ${name} +spec: + parametersRef: + group: networking.gke.io + kind: GKENetworkParamSet + name: ${name} + type: Device diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf new file mode 100644 index 0000000000..9e9ed98ed3 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf @@ -0,0 +1,61 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project that hosts the existing cluster" + type = string +} + +variable "cluster_name" { + description = "Name of the existing cluster" + type = string +} + +variable "region" { + description = "Region in which to search for the cluster" + type = string +} + +variable "additional_networks" { + description = "Additional network interface details for GKE, if any. Providing additional networks creates relevat network objects on the cluster." + default = [] + type = list(object({ + network = string + subnetwork = string + subnetwork_project = string + network_ip = string + nic_type = string + stack_type = string + queue_count = number + access_config = list(object({ + nat_ip = string + network_tier = string + })) + ipv6_access_config = list(object({ + network_tier = string + })) + alias_ip_range = list(object({ + ip_cidr_range = string + subnetwork_range_name = string + })) + })) +} + +variable "rdma_subnetwork_name_prefix" { + description = "Prefix of the RDMA subnetwork names" + default = null + type = string +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf new file mode 100644 index 0000000000..1e458a6b4c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "> 5.0" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:pre-existing-gke-cluster/v1.57.0" + } + + required_version = ">= 1.3" +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/README.md b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/README.md new file mode 100644 index 0000000000..62407666cd --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/README.md @@ -0,0 +1,348 @@ +## Description + +This module creates a startup script that will execute a list of runners in the +order they are specified. The runners are copied to a GCS bucket at deployment +time and then copied into the VM as they are executed after startup. + +Each runner receives the following attributes: + +- `destination`: (Required) The name of the file at the destination VM. If an + absolute path is provided, the file will be copied to that path, otherwise + the file will be created in a temporary folder and deleted once the startup + script runs. +- `type`: (Required) The type of the runner, one of the following: + - `shell`: The runner is a shell script and will be executed once copied to + the destination VM. + - `ansible-local`: The runner is an ansible playbook and will run on the VM + with the following command line flags: + + ```shell + ansible-playbook --connection=local --inventory=localhost, \ + --limit localhost <> + ``` + + - `data`: The data or file specified will be copied to `<>`. No + action will be performed after the data is staged. This data can be used by + subsequent runners or simply made available on the VM for later use. +- `content`: (Optional) Content to be uploaded and, if `type` is + either `shell` or `ansible-local`, executed. Must be defined if `source` is + not. +- `source`: (Optional) A path to the file or data you want to upload. Must be + defined if `content` is not. The source path is relative to the deployment + group directory. To ensure correctness of path use `ghpc_stage` function, that + would copy referenced file to the deployment group directory. For example: + + ```yaml + source: $(ghpc_stage("path/to/file")) + ``` + + For more examples with context, see the + [example blueprint snippet](#example). To reference any other source file, an + absolute path must be used. + +- `args`: (Optional) Arguments to be passed to `shell` or `ansible-local` + runners. For `shell` runners, these will be passed as arguments to the script + when it is executed. For `ansible-local` runners, they will be appended to + a list of default arguments that invoke `ansible-playbook` on the localhost. + Therefore`args` should not include any arguments that alter this behavior, + such as `--connection`, `--inventory`, or `--limit`. + +### Runner dependencies + +`ansible-local` runners require Ansible to be installed in the VM before +running. To support other playbook runners in the Cluster Toolkit, we install +version 2.11 of `ansible-core` as well as the larger package of collections +found in `ansible` version 4.10.0. + +If an `ansible-local` runner is found in the list supplied to this module, +a script to install Ansible will be prepended to the list of runners. This +behavior can be disabled by setting `var.prepend_ansible_installer` to `false`. +This script will do the following at VM startup: + +- Install system-wide python3 if not already installed using system package + managers (yum, apt-get, etc) +- Install `python3-distutils` system-wide in debian and ubuntu based + environments. This can be a missing dependency on system installations of + python3 for installing and upgrading pip. +- Install system-wide pip3 if not already installed and upgrade pip3 if the + version is not at least 18.0. +- Install and create a virtual environment located at `/usr/local/ghpc-venv`. +- Install ansible into this virtual environment if the current version of + ansible is not version 2.11 or higher. + +To use the virtual environment created by this script, you can activate it by +running the following command on the VM: + +```shell +source /usr/local/ghpc-venv/bin/activate +``` + +You may also need to provide the correct python interpreter as the python3 +binary in the virtual environment. This can be done by adding the following flag +when calling `ansible-playbook`: + +```shell +-e ansible_python_interpreter=/usr/local/ghpc-venv/bin/activate +``` + +> **_NOTE:_** ansible-playbook and other ansible command line tools will only be +> accessible from the command line (and in your PATH variable) after activating +> this environment. + +### Staging the runners + +Runners will be uploaded to a +[GCS bucket](https://cloud.google.com/storage/docs/creating-buckets). This +bucket will be created by this module and named as +`${var.deployment_name}-startup-scripts-${random_id}`. VMs using the startup +script created by this module will pull the runners content from a GCS bucket +and therefore must have access to GCS. + +> **_NOTE:_** To ensure access to GCS, set the following OAuth scope on the +> instance using the startup scripts: +> `https://www.googleapis.com/auth/devstorage.read_only`. +> +> This is set as a default scope in the [vm-instance], +> [schedMD-slurm-on-gcp-login-node] and [schedMD-slurm-on-gcp-controller] +> modules + +[vm-instance]: ../../compute/vm-instance/README.md +[schedMD-slurm-on-gcp-login-node]: ../../../community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md +[schedMD-slurm-on-gcp-controller]: ../../../community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md + +### Tracking startup script execution + +For more information on how to use startup scripts on Google Cloud Platform, +please refer to +[this document](https://cloud.google.com/compute/docs/instances/startup-scripts/linux). + +To debug startup scripts from a Linux VM created with startup script generated +by this module: + +```shell +sudo DEBUG=1 google_metadata_script_runner startup +``` + +To view outputs from a Linux startup script, run: + +```shell +sudo journalctl -u google-startup-scripts.service +``` + +### Monitoring Agent Installation + +This `startup-script` module has several options for installing a Google +monitoring agent. There are two relevant settings: `install_stackdriver_agent` +and `install_cloud_ops_agent`. + +The _Stackdriver Agent_ also called the _Legacy Cloud Monitoring Agent_ provides +better performance under some HPC workloads. While official documentation +recommends using the _Cloud Ops Agent_, it is recommended to use +`install_stackdriver_agent` when performance is important. + +#### Stackdriver Agent Installation + +If an image or machine already has Cloud Ops Agent installed and you would like +to instead use the Stackdriver Agent, the following script will remove the Cloud +Ops Agent and install the Stackdriver Agent. + +```bash +# Remove Cloud Ops Agent +sudo systemctl stop google-cloud-ops-agent.service +sudo systemctl disable google-cloud-ops-agent.service +curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh +sudo bash add-google-cloud-ops-agent-repo.sh --uninstall +sudo bash add-google-cloud-ops-agent-repo.sh --remove-repo + +# Install Stackdriver Agent +curl -sSO https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh +sudo bash add-monitoring-agent-repo.sh --also-install +curl -sSO https://dl.google.com/cloudagents/add-logging-agent-repo.sh +sudo bash add-logging-agent-repo.sh --also-install +sudo service stackdriver-agent start +``` + +#### Cloud Ops Agent Installation + +If an image or machine already has the Stackdriver Agent installed and you would +like to instead use the Cloud Ops Agent, the following script will remove the +Stackdriver Agent and install the Cloud Ops Agent. + +```bash +# UnInstall Stackdriver Agent + +sudo systemctl stop stackdriver-agent.service +sudo systemctl disable stackdriver-agent.service +curl -sSO https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh +sudo dpkg --configure -a +sudo bash add-monitoring-agent-repo.sh --uninstall +sudo bash add-monitoring-agent-repo.sh --remove-repo + +# Install ops-agent + +curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh +sudo bash add-google-cloud-ops-agent-repo.sh --also-install +sudo service google-cloud-ops-agent start +``` + +As a reminder, this should be in a startup script, which should run on all +Compute nodes via the `compute_startup_script` on the controller. + +#### Testing Installation + +You can test if one of the agents is running using the following commands: + +```bash +# For Cloud Ops Agent +$ sudo systemctl is-active google-cloud-ops-agent"*" +active +active +active +active + +# For Legacy Monitoring and Logging Agents +$ sudo service stackdriver-agent status +stackdriver-agent is running [ OK ] +$ sudo service google-fluentd status +google-fluentd is running [ OK ] +``` + +For official documentation see troubleshooting docs: + +- [Cloud Ops Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/troubleshoot-install-startup) +- [Legacy Monitoring Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/monitoring/troubleshooting) +- [Legacy Logging Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/logging/troubleshooting) + +### Example + +```yaml +- id: startup + source: modules/scripts/startup-script + settings: + runners: + # Some modules such as filestore have runners as outputs for convenience: + - $(homefs.install_nfs_client_runner) + # These runners can still be created manually: + # - type: shell + # destination: "modules/filestore/scripts/install_nfs_client.sh" + # source: "modules/filestore/scripts/install_nfs_client.sh" + - type: ansible-local + destination: "modules/filestore/scripts/mount.yaml" + source: "modules/filestore/scripts/mount.yaml" + - type: data + source: /tmp/foo.tgz + destination: /tmp/bar.tgz + - type: shell + destination: "decompress.sh" + content: | + #!/bin/sh + echo $2 + tar zxvf /tmp/$1 -C / + args: "bar.tgz 'Expanding file'" + +- id: compute-cluster + source: modules/compute/vm-instance + use: [homefs, startup] +``` + +In the above example, a new GCS bucket is created to upload the startup-scripts. +But in the case where the user wants to reuse existing GCS bucket or folder, +they are able to do so by using the `gcs_bucket_path` as shown in the below example + +```yaml +- id: startup + source: modules/scripts/startup-script + settings: + gcs_bucket_path: gs://user-test-bucket/folder1/folder2 + install_stackdriver_agent: true + +- id: compute-cluster + source: modules/compute/vm-instance + use: [startup] +``` + +## License + + +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5 | +| [google](#requirement\_google) | >= 3.83 | +| [local](#requirement\_local) | >= 2.0.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 3.83 | +| [local](#provider\_local) | >= 2.0.0 | +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket.configs_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | +| [google_storage_bucket_iam_binding.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_binding) | resource | +| [google_storage_bucket_object.scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [local_file.debug_file](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [ansible\_virtualenv\_path](#input\_ansible\_virtualenv\_path) | Virtual environment path in which to install Ansible | `string` | `"/usr/local/ghpc-venv"` | no | +| [bucket\_viewers](#input\_bucket\_viewers) | Additional service accounts or groups, users, and domains to which to grant read-only access to startup-script bucket (leave unset if using default Compute Engine service account) | `list(string)` | `[]` | no | +| [configure\_ssh\_host\_patterns](#input\_configure\_ssh\_host\_patterns) | If specified, it will automate ssh configuration by:
- Defining a Host block for every element of this variable and setting StrictHostKeyChecking to 'No'.
Ex: "hpc*", "hpc01*", "ml*"
- The first time users log-in, it will create ssh keys that are added to the authorized keys list
This requires a shared /home filesystem and relies on specifying the right prefix. | `list(string)` | `[]` | no | +| [debug\_file](#input\_debug\_file) | Path to an optional local to be written with 'startup\_script'. | `string` | `null` | no | +| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used to name GCS bucket for startup scripts. | `string` | n/a | yes | +| [docker](#input\_docker) | Install and configure Docker | 
object({
    enabled        = optional(bool, false)
    world_writable = optional(bool, false)
    daemon_config  = optional(string, "")
  })
| 
{
  "enabled": false
}
| no | +| [enable\_docker\_world\_writable](#input\_enable\_docker\_world\_writable) | DEPRECATED: use var.docker | `bool` | `null` | no | +| [enable\_gpu\_network\_wait\_online](#input\_enable\_gpu\_network\_wait\_online) | Enable a SystemD unit that blocks execution of startup-scripts until after all network interfaces are online. (Works on reboots or boots of an image built using this solution) | `bool` | `false` | no | +| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | `null` | no | +| [http\_no\_proxy](#input\_http\_no\_proxy) | Domains for which to disable http\_proxy behavior. Honored only if var.http\_proxy is set | `string` | `".google.com,.googleapis.com,metadata.google.internal,localhost,127.0.0.1"` | no | +| [http\_proxy](#input\_http\_proxy) | Web (http and https) proxy configuration for pip, apt, and yum/dnf and interactive shells | `string` | `""` | no | +| [install\_ansible](#input\_install\_ansible) | Run Ansible installation script if either set to true or unset and runner of type 'ansible-local' are used. | `bool` | `null` | no | +| [install\_cloud\_ops\_agent](#input\_install\_cloud\_ops\_agent) | Warning: Consider using `install_stackdriver_agent` for better performance. Run Google Ops Agent installation script if set to true. | `bool` | `false` | no | +| [install\_cloud\_rdma\_drivers](#input\_install\_cloud\_rdma\_drivers) | If true, will install and reload Cloud RDMA drivers. Currently only supported on Rocky Linux 8. Should not be enabled if using the HPC VM Image. | `bool` | `false` | no | +| [install\_docker](#input\_install\_docker) | DEPRECATED: use var.docker. | `bool` | `null` | no | +| [install\_stackdriver\_agent](#input\_install\_stackdriver\_agent) | Run Google Stackdriver Agent installation script if set to true. Preferred over ops agent for performance. | `bool` | `false` | no | +| [labels](#input\_labels) | Labels for the created GCS bucket. Key-value pairs. | `map(string)` | n/a | yes | +| [local\_ssd\_filesystem](#input\_local\_ssd\_filesystem) | Create and mount a filesystem from local SSD disks (data will be lost if VMs are powered down without enabling migration); enable by setting mountpoint field to a valid directory path. | 
object({
    fs_type     = optional(string, "ext4")
    mountpoint  = optional(string, "")
    permissions = optional(string, "0755")
  })
| 
{
  "fs_type": "ext4",
  "mountpoint": "",
  "permissions": "0755"
}
| no | +| [managed\_lustre](#input\_managed\_lustre) | Configure Managed Lustre (assumes driver already installed) | 
object({
    enabled = optional(bool, false)
    port    = optional(number, 988)
  })
| 
{
  "enabled": false,
  "port": 988
}
| no | +| [prepend\_ansible\_installer](#input\_prepend\_ansible\_installer) | DEPRECATED. Use `install_ansible=false` to prevent ansible installation. | `bool` | `null` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | +| [runners](#input\_runners) | List of runners to run on remote VM.
Runners can be of type ansible-local, shell or data.
A runner must specify one of 'source' or 'content'.
All runners must specify 'destination'. If 'destination' does not include a
path, it will be copied in a temporary folder and deleted after running.
Runners may also pass 'args', which will be passed as argument to shell runners only. | `list(map(string))` | `[]` | no | +| [set\_ofi\_cloud\_rdma\_tunables](#input\_set\_ofi\_cloud\_rdma\_tunables) | Controls whether to enable specific OFI environment variables for workloads using Cloud RDMA networking. Should be false for non-RDMA workloads. | `bool` | `false` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [compute\_startup\_script](#output\_compute\_startup\_script) | script to load and run all runners, as a string value. Targets the inputs for the slurm controller. | +| [controller\_startup\_script](#output\_controller\_startup\_script) | script to load and run all runners, as a string value. Targets the inputs for the slurm controller. | +| [startup\_script](#output\_startup\_script) | script to load and run all runners, as a string value. | + diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml new file mode 100644 index 0000000000..02c449c7cb --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml @@ -0,0 +1,37 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Configure ssh between nodes + become: true + hosts: localhost + vars: + ssh_config_path: "/etc/ssh/ssh_config" + bashrc: "{{ '/etc/bashrc' if ansible_facts['os_family'] == 'RedHat' else '/etc/bash.bashrc' }}" + setup_ssh_script: "/bin/bash /usr/local/ghpc/setup-ssh-keys.sh" + tasks: + - name: "Set StrictHostKeyChecking to no" + ansible.builtin.blockinfile: + path: "{{ ssh_config_path }}" + block: | + Host "{{ item }}" + StrictHostKeyChecking no + marker: "# {mark} ANSIBLE MANAGED BLOCK {{item}}" + loop: "{{ host_name_prefix }}" + - name: "Create ssh keys in .bashrc if not already done" + ansible.builtin.lineinfile: + path: "{{ bashrc }}" + regexp: '^{{ setup_ssh_script }}' + line: "{{ setup_ssh_script }}" diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh new file mode 100644 index 0000000000..38c7ff9b5c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +web_proxy="${1:-}" +if [ -z "$web_proxy" ]; then + echo "Error: must provide 1 argument identifying http/https proxy" + exit 1 +fi + +# configure pip to use proxy +PIP_CONF=/etc/pip.conf +if [ ! -f "$PIP_CONF" ]; then + cat <<-EOF >"$PIP_CONF" + [global] + proxy=$web_proxy + EOF +fi + +# configure yum or dnf to use proxy +if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || + [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + YUM_CONF="/etc/yum.conf" + if ! grep -q '^proxy=.*' "$YUM_CONF"; then + sed --follow-symlinks -i.bak "/^\[main]/a proxy=$web_proxy" "$YUM_CONF" + else + sed --follow-symlinks -i.bak "s,proxy=.*,proxy=$web_proxy," "$YUM_CONF" + fi +fi + +# configure apt to use proxy +if [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release 2>/dev/null || + grep -qi ubuntu /etc/os-release 2>/dev/null; then + APT_CONF_PROXY="/etc/apt/apt.conf.d/99proxy.conf" + if [ ! -f "$APT_CONF_PROXY" ]; then + cat <<-EOF >"$APT_CONF_PROXY" + Acquire::http::Proxy "$web_proxy"; + Acquire::https::Proxy "$web_proxy"; + EOF + fi +fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh new file mode 100644 index 0000000000..682e1352a1 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script applies fixes to VMs that must occur early in boot. For example, +# when yum or apt repositories are misconfigured, preventing most package +# operations from completing successfully. + +source /etc/os-release + +if [[ "$PRETTY_NAME" == "CentOS Linux 7 (Core)" ]]; then + echo "Applying hotfixes for CentOS 7" + if grep -q '^mirrorlist' /etc/yum.repos.d/CentOS-Base.repo; then + echo "Removing mirrorlist from default CentOS 7 repositories" + sed -i '/^mirrorlist/d' /etc/yum.repos.d/CentOS-Base.repo + fi + if grep -q '^#baseurl=http://mirror.centos.org' /etc/yum.repos.d/CentOS-Base.repo; then + echo "Reconfiguring default CentOS 7 repositories to use CentOS Vault" + sed -i 's,^#baseurl=http://mirror.centos.org/,baseurl=http://vault.centos.org/,' /etc/yum.repos.d/CentOS-Base.repo + fi +fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh new file mode 100644 index 0000000000..3a29ae808f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh @@ -0,0 +1,73 @@ +#! /bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Given a url and filename, download an object to the vardir. When the installed +# version of gcloud is >=402.0.0 (Sept. 2022), then gcloud storage is used to +# fetch from the bucket. Otherwise gsutil is used. Note, the service account for +# the instance must be properly configured with a role having authorization to +# get objects from the bucket. +# +# This function is intended for single file downloads and no attempt is made to +# verify the checksum other than the default behavior of gcloud or gsutil. +# +# This function has no other platform dependencies other than gcloud / gsutil. + +# This code originated from: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 +stdlib::get_from_bucket() { + local OPTIND opt url fname dir="${VARDIR:-/var/lib/startup}" + while getopts ":u:f:d:" opt; do + case "${opt}" in + u) url="${OPTARG}" ;; + f) fname="${OPTARG}" ;; + d) dir="${OPTARG}" ;; + :) + stdlib::mandatory_argument -n stdlib::get_from_bucket -f "$OPTARG" + return "${E_MISSING_MANDATORY_ARG}" + ;; + *) + stdlib::error 'Usage: stdlib::get_from_bucket -u -f -d ' + stdlib::info 'For example: stdlib::get_from_bucket -u gs://mybucket/foo.tgz -d /var/tmp' + return "${E_UNKNOWN_ARG}" + ;; + esac + done + # Trivially compute the filename from the URL if unspecified. + if [[ -z ${fname} ]]; then + fname=${url##*/} + stdlib::debug "Computed filename='${fname}' given URL." + fi + [[ -d ${dir} ]] || mkdir "${dir}" + local attempt=0 + local max_retries=7 + # store gcs command as array and then split when called by stdlib::cmd + if stdlib::cmd gcloud help storage cp &>/dev/null; then + gcs_command=(gcloud storage cp --no-user-output-enabled) + else + gcs_command=(gsutil -q cp) + fi + while [[ $attempt -le $max_retries ]]; do + if [[ $attempt -gt 0 ]]; then + local wait=$((2 ** attempt)) + stdlib::error "Retry attempt ${attempt} of ${max_retries} with exponential backoff: ${wait} seconds." + sleep $wait + fi + if stdlib::cmd "${gcs_command[@]}" "${url}" "${dir}/${fname}"; then + break + else + stdlib::error "${gcs_command[*]} reported non-zero exit code fetching ${url}." + ((attempt++)) + fi + done +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh new file mode 100644 index 0000000000..eac2b2e32a --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh @@ -0,0 +1,247 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex +REQ_ANSIBLE_VERSION=2.15 +REQ_ANSIBLE_PIP_VERSION=8.7.0 +REQ_PIP_WHEEL_VERSION=0.45.1 +REQ_PIP_SETUPTOOLS_VERSION=80.8.0 +REQ_PIP_MAJOR_VERSION=25 +REQ_PYTHON3_VERSION=9 + +apt_wait() { + while fuser /var/lib/apt/lists/lock >/dev/null 2>&1; do + echo "Sleeping for apt lists lock" + sleep 3 + done +} + +# Installs any dependencies needed for python based on the OS +install_python_deps() { + # this file is present on both Debian and Ubuntu OSes + if [ -f /etc/debian_version ]; then + apt_wait + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get install -o DPkg::Lock::Timeout=600 -y python3-setuptools python3-venv + fi +} + +# Gets the name of the python executable for python starting with python3, then +# checking python. Sets the variable to an empty string if neither are found. +get_python_path() { + python_path="" + if command -v python3 1>/dev/null; then + python_path=$(command -v python3) + elif command -v python 1>/dev/null; then + python_path=$(command -v python) + fi +} + +# Returns the python major version. If provided, it will use the first argument +# as the python executable, otherwise it will default to simply "python". +get_python_major_version() { + python_path=${1:-python} + python_major_version=$(${python_path} -c "import sys; print(sys.version_info.major)") +} + +# Returns the python minor version. If provided, it will use the first argument +# as the python executable, otherwise it will default to simply "python". +get_python_minor_version() { + python_path=${1:-python} + python_minor_version=$(${python_path} -c "import sys; print(sys.version_info.minor)") +} + +# Install python3 with the yum package manager. Updates python_path to the +# newly installed packaged. +install_python3_dnf() { + major_version=$(rpm -E "%{rhel}") + set -- "--disablerepo=*" "--enablerepo=baseos,appstream" + if grep -qi 'ID="rhel"' /etc/os-release; then + # Do not set --disablerepo / --enablerepo on RedHat, due to + # complex repo names; clear array + set -- + fi + # On Rocky Linux 9, Python 3.9 is installed by default but this + # has already been dropped by ansible-core for control nodes. + # https://docs.ansible.com/ansible/latest/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix + # Python 3.12 aligns with RHEL 10 default (GA: 13 May 2025) where + # it is available as "python3*" but must be named explicitly on + # older releases. It also ensures longer support for Ansible. + if [ "${major_version}" -lt "10" ]; then + dnf install "$@" -y python3.12 python3.12-pip + python_path=$(command -v python3.12) + else + dnf install "$@" -y python3 python3-pip + python_path=$(command -v python3) + fi +} + +# Install python3 with the apt package manager. Updates python_path to the +# newly installed packaged. +install_python3_apt() { + apt_wait + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get install -o DPkg::Lock::Timeout=600 -y python3 python3-setuptools python3-pip python3-venv + python_path=$(command -v python3) +} + +install_python3() { + if [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || + [ -f /etc/system-release ]; then + install_python3_dnf + elif [ -f /etc/debian_version ]; then + install_python3_apt + else + echo "Error: Unsupported Distribution" + return 1 + fi +} + +# Install pip3 with the dnf package manager. Updates python_path to the +# newly installed packaged. +install_pip3_dnf() { + major_version=$(rpm -E "%{rhel}") + set -- "--disablerepo=*" "--enablerepo=baseos,appstream" + if grep -qi 'ID="rhel"' /etc/os-release; then + # Do not set --disablerepo / --enablerepo on RedHat, due to complex repo names + # clear array + set -- + fi + # Python 3.12 aligns with RHEL 10 default (GA: 13 May 2025) where + # it is available as "python3*" but must be named explicitly on + # older releases. It also ensures longer support for Ansible. + if [ "${major_version}" -lt "10" ]; then + dnf install "$@" -y python3.12-pip + else + dnf install "$@" -y python3-pip + fi +} + +# Install pip3 with the apt package manager. Updates python_path to the +# newly installed packaged. +install_pip3_apt() { + apt_wait + apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label + apt-get install -o DPkg::Lock::Timeout=600 -y python3-pip +} + +install_pip3() { + if [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || + [ -f /etc/system-release ]; then + install_pip3_dnf + elif [ -f /etc/debian_version ]; then + install_pip3_apt + else + echo "Error: Unsupported Distribution" + return 1 + fi +} + +main() { + if [ $# -gt 1 ]; then + echo "Error: provide only 1 optional argument identifying virtual environment path for Ansible" + return 1 + fi + + venv_path="${1:-/usr/local/ghpc-venv}" + + # Get the python3 executable, or install it if not found + get_python_path + get_python_major_version "${python_path}" + get_python_minor_version "${python_path}" + if [ "${python_path}" = "" ] || [ "${python_major_version}" = "2" ] || [ "${python_minor_version}" -lt "${REQ_PYTHON3_VERSION}" ]; then + if ! install_python3; then + return 1 + fi + get_python_major_version "${python_path}" + get_python_minor_version "${python_path}" + else + install_python_deps + fi + + # Install OS-packaged pip + if ! ${python_path} -m pip --version 2>/dev/null; then + if ! install_pip3; then + return 1 + fi + fi + + # Create pip virtual environment for Cluster Toolkit + ${python_path} -m venv "${venv_path}" --copies + venv_python_path=${venv_path}/bin/python3 + + # Upgrade pip if necessary + pip_version=$(${venv_python_path} -m pip --version | sed -nr 's/^pip ([0-9]+\.[0-9]+).*$/\1/p') + pip_major_version=$(echo "${pip_version}" | cut -d '.' -f 1) + if [ "${pip_major_version}" -lt "${REQ_PIP_MAJOR_VERSION}" ]; then + ${venv_python_path} -m pip install --upgrade pip + fi + + # upgrade wheel if necessary + wheel_pkg=$(${venv_python_path} -m pip list --format=freeze | grep "^wheel" || true) + if [ "$wheel_pkg" != "wheel==${REQ_PIP_WHEEL_VERSION}" ]; then + ${venv_python_path} -m pip install -U wheel==${REQ_PIP_WHEEL_VERSION} + fi + + # upgrade setuptools if necessary + setuptools_pkg=$(${venv_python_path} -m pip list --format=freeze | grep "^setuptools" || true) + if [ "$setuptools_pkg" != "setuptools==${REQ_PIP_SETUPTOOLS_VERSION}" ]; then + ${venv_python_path} -m pip install -U setuptools==${REQ_PIP_SETUPTOOLS_VERSION} + fi + + # configure ansible to always use correct Python binary + if [ ! -f /etc/ansible/ansible.cfg ]; then + mkdir /etc/ansible + cat <<-EOF >/etc/ansible/ansible.cfg + [defaults] + interpreter_python=${venv_python_path} + stdout_callback=debug + stderr_callback=debug + EOF + fi + + # Install ansible + ansible_version="" + if command -v ansible-playbook 1>/dev/null; then + ansible_version=$(ansible-playbook --version 2>/dev/null | sed -nr 's/^ansible-playbook.*([0-9]+\.[0-9]+\.[0-9]+).*/\1/p') + ansible_major_vers=$(echo "${ansible_version}" | cut -d '.' -f 1) + ansible_minor_vers=$(echo "${ansible_version}" | cut -d '.' -f 2) + ansible_req_major_vers=$(echo "${REQ_ANSIBLE_VERSION}" | cut -d '.' -f 1) + ansible_req_minor_vers=$(echo "${REQ_ANSIBLE_VERSION}" | cut -d '.' -f 2) + fi + if [ -z "${ansible_version}" ] || [ "${ansible_major_vers}" -ne "${ansible_req_major_vers}" ] || + [ "${ansible_minor_vers}" -lt "${ansible_req_minor_vers}" ]; then + ${venv_python_path} -m pip install ansible=="${REQ_ANSIBLE_PIP_VERSION}" + fi + while read -r cmd; do + if ! [ -L "/usr/bin/${cmd}" ]; then + ln -s "${venv_path}/bin/${cmd}" "/usr/bin/${cmd}" + fi + done <<-EOF + ansible + ansible-config + ansible-connection + ansible-console + ansible-doc + ansible-galaxy + ansible-inventory + ansible-playbook + ansible-pull + ansible-test + ansible-vault + EOF +} + +main "$@" diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh new file mode 100644 index 0000000000..375792459b --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e -o pipefail + +OS_ID="$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g')" +OS_VERSION="$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g')" +OS_VERSION_MAJOR="$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//')" +REBOOT_FILE="/etc/.rdma_reboot" + +if { [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; } && { [ "${OS_VERSION_MAJOR}" = "8" ]; }; then + KMOD_VERSION="$(dnf list installed | awk '$1 ~ /^kmod-idpf-irdma(\.|$)/ {print $2}')" + + # For images that do not already have Cloud RDMA drivers installed + if [ -z "${KMOD_VERSION}" ] && [ -z "${REBOOT_FILE}" ]; then + sudo dnf update -y + sudo dnf install https://depot.ciq.com/public/files/gce-accelerator/irdma-kernel-modules-el8-x86_64/irdma-repos.rpm -y + sudo dnf install kmod-idpf-irdma rdma-core libibverbs-utils librdmacm-utils infiniband-diags perftest -y + sudo touch "${REBOOT_FILE}" + reboot + fi + echo "This image has IRDMA packages already installed, exiting." + exit 0 +else + echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. Cloud RDMA Drivers are only supported on Rocky Linux 8." + exit 1 +fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_docker.yml b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_docker.yml new file mode 100644 index 0000000000..af251014b8 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_docker.yml @@ -0,0 +1,167 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Install and configure Docker + hosts: all + become: true + vars: + docker_data_root: '' + docker_daemon_config: '' + enable_docker_world_writable: false + is_rocky_linux: "{{ ansible_distribution in ['Rocky', 'Rocky Linux'] }}" + tasks: + - name: Gather distribution facts + ansible.builtin.setup: + gather_subset: platform + - name: Check if docker is installed + ansible.builtin.stat: + path: /usr/bin/docker + register: docker_binary + - name: Add Docker GPG key (Rocky) + ansible.builtin.rpm_key: + state: present + key: https://download.docker.com/linux/rhel/gpg + when: + - is_rocky_linux + - not docker_binary.stat.exists + register: add_docker_key + retries: 30 + delay: 10 + until: add_docker_key is succeeded + - name: Add Docker repository for Rocky Linux + ansible.builtin.command: > + dnf config-manager --add-repo + https://download.docker.com/linux/rhel/docker-ce.repo + when: + - is_rocky_linux + - not docker_binary.stat.exists + notify: Refresh dnf cache + - name: Install Docker Engine and related packages on Rocky + ansible.builtin.package: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + state: present + update_cache: yes + when: + - is_rocky_linux + - not docker_binary.stat.exists + - name: Ensure Docker SDK for Python is installed + block: + - name: Install python3-pip + ansible.builtin.package: + name: python3-pip + state: present + - name: Install Docker SDK for Python + ansible.builtin.pip: + name: docker + executable: /usr/local/ghpc-venv/bin/pip3 + when: is_rocky_linux + - name: Download Docker Installer (get.docker.com) + ansible.builtin.get_url: + url: https://get.docker.com + dest: /tmp/get-docker.sh + owner: root + group: root + mode: '0644' + when: + - not is_rocky_linux + - not docker_binary.stat.exists + - name: Install Docker + ansible.builtin.command: sh /tmp/get-docker.sh + register: docker_installed + changed_when: docker_installed.rc != 0 + when: + - not is_rocky_linux + - not docker_binary.stat.exists + - name: Create Docker daemon configuration + ansible.builtin.copy: + dest: /etc/docker/daemon.json + mode: '0644' + content: '{{ docker_daemon_config }}' + validate: /usr/bin/dockerd --validate --config-file %s + when: docker_daemon_config + notify: + - Restart Docker + - name: Create Docker service override directory + ansible.builtin.file: + path: /etc/systemd/system/docker.service.d + state: directory + owner: root + group: root + mode: '0755' + - name: Create Docker service override configuration + ansible.builtin.copy: + dest: /etc/systemd/system/docker.service.d/data-root.conf + mode: '0644' + content: | + [Unit] + {% if docker_data_root %} + RequiresMountsFor={{ docker_data_root }} + {% endif %} + After=mount-localssd-raid.service + - name: Create Docker socket override directory + ansible.builtin.file: + path: /etc/systemd/system/docker.socket.d + state: directory + owner: root + group: root + mode: '0755' + when: enable_docker_world_writable + - name: Create Docker socket override configuration + ansible.builtin.copy: + dest: /etc/systemd/system/docker.socket.d/world-writable.conf + mode: '0644' + content: | + [Socket] + SocketMode=0666 + when: enable_docker_world_writable + notify: + - Reload SystemD + - Recreate Docker socket + - name: Delete Docker socket override configuration + ansible.builtin.file: + path: /etc/systemd/system/docker.socket.d/world-writable.conf + state: absent + when: not enable_docker_world_writable + notify: + - Reload SystemD + - Recreate Docker socket + + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true + - name: Recreate Docker socket + ansible.builtin.service: + name: docker.socket + state: restarted + - name: Restart Docker + ansible.builtin.service: + name: docker.service + state: restarted + - name: Refresh dnf cache + ansible.builtin.dnf: + update_cache: yes + + post_tasks: + - name: Start Docker + ansible.builtin.service: + name: docker.service + state: started + enabled: true diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml new file mode 100644 index 0000000000..7f3dda3d24 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml @@ -0,0 +1,54 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Wait until all network interfaces are online + hosts: all + become: true + tasks: + - name: Create SystemD service for A3 High networking + when: ansible_os_family == "Debian" + ansible.builtin.copy: + dest: /etc/systemd/system/delay-a3-high.service + owner: root + group: root + mode: "0644" + content: | + [Unit] + Description=Delay A3 High boot until all network interfaces are routable + After=network-online.target + Wants=network-online.target + Before=google-startup-scripts.service + + [Service] + ExecCondition=/bin/bash -c '/usr/bin/curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetata/v1/instance/machine-type | grep -q "/a3-highgpu-8g$"' + ExecStart=/usr/lib/systemd/systemd-networkd-wait-online -i enp0s12 -i enp6s0 -i enp12s0 -i enp134s0 -i enp140s0 -o routable --timeout=180 + ExecStartPost=/bin/sleep 30 + + [Install] + WantedBy=multi-user.target + notify: + - Reload SystemD + - Enable A3 High delay + handlers: + - name: Reload SystemD + ansible.builtin.systemd: + daemon_reload: true + post_tasks: + - name: Enable A3 High delay + # by the time this startup-script executes, we don't care if it's started + # just enabled + ansible.builtin.systemd_service: + name: delay-a3-high + enabled: true diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml new file mode 100644 index 0000000000..94699471bb --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml @@ -0,0 +1,33 @@ +# Copyright 2025 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure Managed Lustre (assumes driver already installed) + hosts: all + become: true + vars: + default_lustre_port: 988 + managed_lustre_port: "{{ default_lustre_port }}" + tasks: + # Ideally changes to this file would also trigger an execution of lnetctl + # command to update accept_port but it is unclear if lnetctl supports this. + - name: Update lnet to use non-default port + when: managed_lustre_port | int != {{ default_lustre_port }} + ansible.builtin.copy: + owner: root + group: root + mode: '0644' + dest: /etc/modprobe.d/lnet.conf + content: | + options lnet accept_port={{ managed_lustre_port | int }} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh new file mode 100644 index 0000000000..eb4bf899b8 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e -o pipefail + +LEGACY_MONITORING_PACKAGE='stackdriver-agent' +LEGACY_MONITORING_SCRIPT_URL='https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh' +LEGACY_LOGGING_PACKAGE='google-fluentd' +LEGACY_LOGGING_SCRIPT_URL='https://dl.google.com/cloudagents/add-logging-agent-repo.sh' + +OPSAGENT_PACKAGE='google-cloud-ops-agent' +OPSAGENT_SCRIPT_URL='https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh' + +ops_or_legacy="${1:-legacy}" + +fail() { + echo >&2 "[$(date +'%Y-%m-%dT%H:%M:%S%z')] $*" + exit 1 +} + +handle_debian() { + is_legacy_monitoring_installed() { + dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${LEGACY_MONITORING_PACKAGE} | + grep "${LEGACY_MONITORING_PACKAGE} is installed" + } + + is_legacy_logging_installed() { + dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${LEGACY_LOGGING_PACKAGE} | + grep "${LEGACY_LOGGING_PACKAGE} is installed" + } + + is_legacy_installed() { + is_legacy_monitoring_installed || is_legacy_logging_installed + } + + is_opsagent_installed() { + dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${OPSAGENT_PACKAGE} | + grep "${OPSAGENT_PACKAGE} is installed" + } + + install_with_retry() { + MAX_RETRY=50 + RETRY=0 + until [ ${RETRY} -eq ${MAX_RETRY} ] || curl -s "${1}" | bash -s -- --also-install; do + RETRY=$((RETRY + 1)) + echo "WARNING: Installation of ${1} failed on try ${RETRY} of ${MAX_RETRY}" + sleep 5 + done + if [ $RETRY -eq $MAX_RETRY ]; then + echo "ERROR: Installation of ${1} was not successful after ${MAX_RETRY} attempts." + exit 1 + fi + } + + install_opsagent() { + install_with_retry "${OPSAGENT_SCRIPT_URL}" + } + + install_stackdriver_agent() { + install_with_retry "${LEGACY_MONITORING_SCRIPT_URL}" + install_with_retry "${LEGACY_LOGGING_SCRIPT_URL}" + service stackdriver-agent start + service google-fluentd start + } +} + +handle_redhat() { + is_legacy_monitoring_installed() { + rpm --query --queryformat 'package %{NAME} is installed\n' ${LEGACY_MONITORING_PACKAGE} | + grep "${LEGACY_MONITORING_PACKAGE} is installed" + } + + is_legacy_logging_installed() { + rpm --query --queryformat 'package %{NAME} is installed\n' ${LEGACY_LOGGING_PACKAGE} | + grep "${LEGACY_LOGGING_PACKAGE} is installed" + } + + is_legacy_installed() { + is_legacy_monitoring_installed || is_legacy_logging_installed + } + + is_opsagent_installed() { + rpm --query --queryformat 'package %{NAME} is installed\n' ${OPSAGENT_PACKAGE} | + grep "${OPSAGENT_PACKAGE} is installed" + } + + install_opsagent() { + curl -s "${OPSAGENT_SCRIPT_URL}" | bash -s -- --also-install + } + + install_stackdriver_agent() { + curl -sS "${LEGACY_MONITORING_SCRIPT_URL}" | bash -s -- --also-install + curl -sS "${LEGACY_LOGGING_SCRIPT_URL}" | bash -s -- --also-install + service stackdriver-agent start + service google-fluentd start + } +} + +main() { + if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then + handle_redhat + elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then + handle_debian + else + fail "Unsupported platform." + fi + + # Handle cases that agent is already installed + if [[ -z "$(is_legacy_monitoring_installed)" && -n $(is_legacy_logging_installed) ]] || + [[ -n "$(is_legacy_monitoring_installed)" && -z $(is_legacy_logging_installed) ]]; then + fail "Bad state: legacy agent is partially installed" + elif [[ "${ops_or_legacy}" == "legacy" ]] && is_legacy_installed; then + echo "Legacy agent is already installed" + exit 0 + elif [[ "${ops_or_legacy}" != "legacy" ]] && is_opsagent_installed; then + echo "Ops agent is already installed" + exit 0 + elif is_legacy_installed || is_opsagent_installed; then + fail "Agent is already installed but does not match requested agent of ${ops_or_legacy}" + fi + + # install agent + if [[ "${ops_or_legacy}" == "legacy" ]]; then + echo "Installing legacy monitoring agent (stackdriver)" + install_stackdriver_agent + else + echo "Installing cloud ops agent" + echo "WARNING: cloud ops agent may have a performance impact. Consider using legacy monitoring agent (stackdriver)." + install_opsagent + fi +} + +main diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh new file mode 100644 index 0000000000..738181aafb --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_COMPLETE_FILE="/run/startup_script_msg" + +# Ensure we're in an interactive terminal and not root +if [ -t 1 ] && [ "$(id -u)" -ne 0 ]; then + # Check if the file has contents otherwise skip + if [ -s "$SCRIPT_COMPLETE_FILE" ]; then + echo + cat "$SCRIPT_COMPLETE_FILE" + echo + fi +fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml new file mode 100644 index 0000000000..d94aac81fd --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml @@ -0,0 +1,100 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Configure local SSDs + become: true + hosts: localhost + vars: + raid_name: localssd + array_dev: /dev/md/{{ raid_name }} + fstype: ext4 + interface: nvme + mode: '0755' + mountpoint: /mnt/{{ raid_name }} + tasks: + - name: Get local SSD devices + ansible.builtin.find: + file_type: link + path: /dev/disk/by-id + patterns: google-local-{{ "nvme-" if interface == "nvme" else "" }}ssd-* + register: local_ssd_devices + + - name: Exit if zero local ssd found + ansible.builtin.meta: end_play + when: local_ssd_devices.files | length == 0 + + - name: Install mdadm + ansible.builtin.package: + name: mdadm + state: present + + # this service will act during the play and upon reboots to ensure that local + # SSD volumes are always assembled into a RAID and re-formatted if necessary; + # there are many scenarios where a VM can be stopped or migrated during + # maintenance and the contents of local SSD will be discarded + - name: Install service to create local SSD RAID and format it + ansible.builtin.copy: + dest: /etc/systemd/system/create-localssd-raid.service + mode: 0644 + content: | + [Unit] + After=local-fs.target + Before=slurmd.service docker.service + ConditionPathExists=!{{ array_dev }} + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=/usr/bin/bash -c "/usr/sbin/mdadm --create {{ array_dev }} --name={{ raid_name }} --homehost=any --level=0 --raid-devices={{ local_ssd_devices.files | length }} /dev/disk/by-id/google-local-nvme-ssd-*{{ " --force" if local_ssd_devices.files | length == 1 else "" }}" + ExecStartPost=/usr/sbin/mkfs -t {{ fstype }}{{ " -m 0" if fstype == "ext4" else "" }} {{ array_dev }} + + [Install] + WantedBy=slurmd.service docker.service + + - name: Create RAID array and format + ansible.builtin.systemd: + name: create-localssd-raid.service + state: started + enabled: true + daemon_reload: true + + - name: Install service to mount local SSD array + ansible.builtin.copy: + dest: /etc/systemd/system/mount-localssd-raid.service + mode: 0644 + content: | + [Unit] + After=local-fs.target create-localssd-raid.service + Before=slurmd.service docker.service + Wants=create-localssd-raid.service + ConditionPathIsMountPoint=!{{ mountpoint }} + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=/usr/bin/systemd-mount -t {{ fstype }} -o discard,defaults,nofail {{ array_dev }} {{ mountpoint }} + ExecStartPost=/usr/bin/chmod {{ mode }} {{ mountpoint }} + ExecStop=/usr/bin/systemd-umount {{ mountpoint }} + + [Install] + WantedBy=slurmd.service docker.service + + - name: Mount RAID array and set permissions + ansible.builtin.systemd: + name: mount-localssd-raid.service + state: started + enabled: true + daemon_reload: true diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh new file mode 100644 index 0000000000..1c8018fb01 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ! -d ~/.ssh/ ]; then + source /usr/local/ghpc-venv/bin/activate + ansible-playbook /usr/local/ghpc/setup-ssh-keys.yml +fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml new file mode 100644 index 0000000000..692896bb9c --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml @@ -0,0 +1,40 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Setup SSH Keys for user + become: false + hosts: localhost + vars: + pub_key_path: "{{ ansible_env.HOME }}/.ssh" + pub_key_file: "{{ pub_key_path }}/id_rsa" + auth_key_file: "{{ pub_key_path }}/authorized_keys" + tasks: + - name: "Create .ssh folder" + ansible.builtin.file: + path: "{{ pub_key_path }}" + state: directory + mode: 0700 + owner: "{{ ansible_user_id }}" + - name: Create keys + community.crypto.openssh_keypair: + path: "{{ pub_key_file }}" + owner: "{{ ansible_user_id }}" + - name: Copy public key to authorized keys + ansible.builtin.copy: + src: "{{ pub_key_file }}.pub" + dest: "{{ auth_key_file }}" + owner: "{{ ansible_user_id }}" + mode: 0644 diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh new file mode 100644 index 0000000000..8ca40bc73f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh @@ -0,0 +1,39 @@ +#! /bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This code contains minor changes from the original: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 + +stdlib::main() { + DELETE_AT_EXIT="$(mktemp -d)" + readonly DELETE_AT_EXIT + + # Initialize state required by other functions, e.g. debug() + stdlib::init + stdlib::debug "Loaded startup-script-stdlib as an executable." + + stdlib::load_config_values + + stdlib::load_runners +} + +# if script is being executed and not sourced. +if [[ ${BASH_SOURCE[0]} == "${0}" ]]; then + stdlib::finish() { + [[ -d ${DELETE_AT_EXIT:-} ]] && rm -rf "${DELETE_AT_EXIT}" + } + trap stdlib::finish EXIT + + stdlib::main "$@" +fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh new file mode 100644 index 0000000000..589a3215ab --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh @@ -0,0 +1,266 @@ +#! /bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This code contains minor changes from the original in: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 + +# Standard library of functions useful for startup scripts. + +# These are outside init_global_vars so logging functions work with the most +# basic case of `source startup-script-stdlib.sh` +readonly SYSLOG_DEBUG_PRIORITY="${SYSLOG_DEBUG_PRIORITY:-syslog.debug}" +readonly SYSLOG_INFO_PRIORITY="${SYSLOG_INFO_PRIORITY:-syslog.info}" +readonly SYSLOG_ERROR_PRIORITY="${SYSLOG_ERROR_PRIORITY:-syslog.error}" +# Global counter of how many times stdlib::init() has been called. +STARTUP_SCRIPT_STDLIB_INITIALIZED=0 + +# Error codes +readonly E_RUN_OR_DIE=5 +readonly E_MISSING_MANDATORY_ARG=9 +readonly E_UNKNOWN_ARG=10 + +SCRIPT_COMPLETE_FILE="/run/startup_script_msg" +SUCCESS_MESSAGE="* NOTICE **: The Cluster Toolkit startup scripts have finished running successfully." +readonly SUCCESS_MESSAGE +ERROR_MESSAGE="** ERROR **: The Cluster Toolkit startup scripts have finished running, but produced an error." +readonly ERROR_MESSAGE +WARNING_MESSAGE="** WARNING **: The Cluster Toolkit startup scripts are currently running." +readonly WARNING_MESSAGE + +stdlib::debug() { + [[ -z ${DEBUG:-} ]] && return 0 + local ds msg + msg="$*" + logger -p "${SYSLOG_DEBUG_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" + [[ -n ${QUIET:-} ]] && return 0 + ds="$(date +"${DATE_FMT}") " + echo -e "${BLUE}${ds}Debug [$$]: ${msg}${NC}" >&2 +} + +stdlib::info() { + local ds msg + msg="$*" + logger -p "${SYSLOG_INFO_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" + [[ -n ${QUIET:-} ]] && return 0 + ds="$(date +"${DATE_FMT}") " + echo -e "${GREEN}${ds}Info [$$]: ${msg}${NC}" >&2 +} + +stdlib::error() { + local ds msg + msg="$*" + ds="$(date +"${DATE_FMT}") " + logger -p "${SYSLOG_ERROR_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" + echo -e "${RED}${ds}Error [$$]: ${msg}${NC}" >&2 +} + +stdlib::announce_runners_start() { + if [ -z "$recursive_proc" ]; then + wall -n "$WARNING_MESSAGE" + echo "$WARNING_MESSAGE" >"$SCRIPT_COMPLETE_FILE" + fi + export recursive_proc=$((${recursive_proc:=0} + 1)) +} + +stdlib::announce_runners_end() { + exit_code=$1 + export recursive_proc=$((${recursive_proc:=0} - 1)) + if [ "$recursive_proc" -le "0" ]; then + if [ "$exit_code" -ne "0" ]; then + wall -n "$ERROR_MESSAGE" + echo "$ERROR_MESSAGE" >"$SCRIPT_COMPLETE_FILE" + else + wall -n "$SUCCESS_MESSAGE" + echo -n "" >"$SCRIPT_COMPLETE_FILE" + fi + fi +} + +# The main initialization function of this library. This should be kept to the +# minimum amount of work required for all functions to operate cleanly. +stdlib::init() { + if [[ ${STARTUP_SCRIPT_STDLIB_INITIALIZED} -gt 0 ]]; then + stdlib::info 'stdlib::init()'" already initialized, no action taken." + return 0 + fi + ((STARTUP_SCRIPT_STDLIB_INITIALIZED++)) || true + stdlib::init_global_vars + stdlib::init_directories + stdlib::debug "stdlib::init(): startup-script-stdlib.sh initialized and ready" +} + +# Initialize global variables. +stdlib::init_global_vars() { + # The program name, used for logging. + readonly PROG="${PROG:-startup-script-stdlib}" + # Date format used for stderr logging. Passed to date + command. + readonly DATE_FMT="${DATE_FMT:-"%a %b %d %H:%M:%S %z %Y"}" + # var directory + readonly VARDIR="${VARDIR:-/var/lib/startup}" + # Override this with file://localhost/tmp/foo/bar in spec test context + readonly METADATA_BASE="${METADATA_BASE:-http://metadata.google.internal}" + + # Color variables + if [[ -n ${COLOR:-} ]]; then + readonly NC='\033[0m' # no color + readonly RED='\033[0;31m' # error + readonly GREEN='\033[0;32m' # info + readonly BLUE='\033[0;34m' # debug + else + readonly NC='' + readonly RED='' + readonly GREEN='' + readonly BLUE='' + fi + + return 0 +} + +stdlib::init_directories() { + if ! [[ -e ${VARDIR} ]]; then + install -d -m 0755 -o 0 -g 0 "${VARDIR}" + fi +} + +## +# Get a metadata key. When used without -o, this function is guaranteed to +# produce no output on STDOUT other than the retrieved value. This is intended +# to support the use case of +# FOO="$(stdlib::metadata_get -k instance/attributes/foo)" +# +# If the requested key does not exist, the error code will be 22 and zero bytes +# written to STDOUT. +stdlib::metadata_get() { + local OPTIND opt key outfile + local metadata="${METADATA_BASE%/}/computeMetadata/v1" + local exit_code + while getopts ":k:o:" opt; do + case "${opt}" in + k) key="${OPTARG}" ;; + o) outfile="${OPTARG}" ;; + :) + stdlib::error "Invalid option: -${OPTARG} requires an argument" + stdlib::metadata_get_usage + return "${E_MISSING_MANDATORY_ARG}" + ;; + *) + stdlib::error "Unknown option: -${opt}" + stdlib::metadata_get_usage + return "${E_UNKNOWN_ARG}" + ;; + esac + done + local url="${metadata}/${key#/}" + + stdlib::debug "Getting metadata resource url=${url}" + if [[ -z ${outfile:-} ]]; then + curl --location --silent --connect-timeout 1 --fail \ + -H 'Metadata-Flavor: Google' "$url" 2>/dev/null + exit_code=$? + else + stdlib::cmd curl --location \ + --silent \ + --connect-timeout 1 \ + --fail \ + --output "${outfile}" \ + -H 'Metadata-Flavor: Google' \ + "$url" + exit_code=$? + fi + case "${exit_code}" in + 22 | 37) + stdlib::debug "curl exit_code=${exit_code} for url=${url}" \ + "(Does not exist)" + ;; + esac + return "${exit_code}" +} + +stdlib::metadata_get_usage() { + stdlib::info 'Usage: stdlib::metadata_get -k ' + stdlib::info 'For example: stdlib::metadata_get -k instance/attributes/startup-config' +} + +# Load configuration values in the spirit of /etc/sysconfig defaults, but from +# metadata instead of the filesystem. +stdlib::load_config_values() { + local config_file + local key="instance/attributes/startup-script-config" + # shellcheck disable=SC2119 + config_file="$(stdlib::mktemp)" + stdlib::metadata_get -k "${key}" -o "${config_file}" + local status=$? + case "$status" in + 0) + stdlib::debug "SUCCESS: Configuration data sourced from $key" + ;; + 22 | 37) + stdlib::debug "no configuration data loaded from $key" + ;; + *) + stdlib::error "metadata_get -k $key returned unknown status=${status}" + ;; + esac + # shellcheck source=/dev/null + source "${config_file}" +} + +# Run a command logging the entry and exit. Intended for system level commands +# and operational debugging. Not intended for use with redirection. This is +# not named run() because bats uses a run() function. +stdlib::cmd() { + local exit_code argv=("$@") + stdlib::debug "BEGIN: stdlib::cmd() command=[${argv[*]}]" + "${argv[@]}" + exit_code=$? + stdlib::debug "END: stdlib::cmd() command=[${argv[*]}] exit_code=${exit_code}" + return $exit_code +} + +# Run a command successfully or exit the program with an error. +stdlib::run_or_die() { + if ! stdlib::cmd "$@"; then + stdlib::error "stdlib::run_or_die(): exiting with exit code ${E_RUN_OR_DIE}." + exit "${E_RUN_OR_DIE}" + fi +} + +# Intended to take advantage of automatic cleanup of startup script library +# temporary files without exporting a modified TMPDIR to child processes, which +# would cause the children to have their TMPDIR deleted out from under them. +# shellcheck disable=SC2120 +stdlib::mktemp() { + TMPDIR="${DELETE_AT_EXIT:-${TMPDIR}}" mktemp "$@" +} + +# Return a nice error message if a mandatory argument is missing. +stdlib::mandatory_argument() { + local OPTIND opt name flag + while getopts ":n:f:" opt; do + case "$opt" in + n) name="${OPTARG}" ;; + f) flag="${OPTARG}" ;; + :) + stdlib::error "Invalid argument: -${OPTARG} requires an argument to stdlib::mandatory_argument()" + return "${E_MISSING_MANDATORY_ARG}" + ;; + *) + stdlib::error "Unknown argument: -${OPTARG}" + stdlib::info "Usage: stdlib::mandatory_argument -n -f " + return "${E_UNKNOWN_ARG}" + ;; + esac + done + stdlib::error "Invalid argument: -${flag} requires an argument to ${name}()." +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/main.tf b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/main.tf new file mode 100644 index 0000000000..d91fb3edb1 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/main.tf @@ -0,0 +1,305 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # This label allows for billing report tracking based on module. + labels = merge(var.labels, { ghpc_module = "startup-script", ghpc_role = "scripts" }) +} + +locals { + monitoring_agent_installer = ( + var.install_cloud_ops_agent || var.install_stackdriver_agent ? + [{ + type = "shell" + source = "${path.module}/files/install_monitoring_agent.sh" + destination = "install_monitoring_agent_automatic.sh" + args = var.install_cloud_ops_agent ? "ops" : "legacy" # install legacy (stackdriver) + }] : + [] + ) + + warnings = [ + { + type = "data" + content = file("${path.module}/files/running-script-warning.sh") + destination = "/etc/profile.d/99-running-script-warning.sh" + } + ] + + configure_ssh = length(var.configure_ssh_host_patterns) > 0 + host_args = { + host_name_prefix = var.configure_ssh_host_patterns + } + + prefix_file = "/tmp/prefix_file.json" + ansible_docker_settings_file = "/tmp/ansible_docker_settings.json" + + docker_config = try(jsondecode(var.docker.daemon_config), {}) + docker_data_root = try(local.docker_config.data-root, null) + + configure_ssh_runners = local.configure_ssh ? [ + { + type = "data" + source = "${path.module}/files/setup-ssh-keys.sh" + destination = "/usr/local/ghpc/setup-ssh-keys.sh" + }, + { + type = "data" + source = "${path.module}/files/setup-ssh-keys.yml" + destination = "/usr/local/ghpc/setup-ssh-keys.yml" + }, + { + type = "data" + content = jsonencode(local.host_args) + destination = local.prefix_file + }, + { + type = "ansible-local" + content = file("${path.module}/files/configure-ssh.yml") + destination = "configure-ssh.yml" + args = "-e @${local.prefix_file}" + } + ] : [] + + proxy_runner = var.http_proxy == "" ? [] : [ + { + type = "data" + destination = "/etc/profile.d/http_proxy.sh" + content = <<-EOT + #!/bin/bash + export http_proxy=${var.http_proxy} + export https_proxy=${var.http_proxy} + export NO_PROXY=${var.http_no_proxy} + EOT + }, + { + type = "shell" + source = "${path.module}/files/configure_proxy.sh" + destination = "configure_proxy.sh" + args = var.http_proxy + } + ] + + ofi_runner = !var.set_ofi_cloud_rdma_tunables ? [] : [ + { + type = "data" + destination = "/etc/profile.d/set_ofi_cloud_rdma_tunables.sh" + content = <<-EOT + #!/bin/bash + export FI_PROVIDER="verbs;ofi_rxm" + export FI_OFI_RXM_USE_RNDV_WRITE=0 + export FI_VERBS_INLINE_SIZE=39 + export I_MPI_FABRICS="shm:ofi" + export FI_UNIVERSE_SIZE=3072 + export I_MPI_ADJUST_ALLTOALL=1 + export I_MPI_ADJUST_IALLTOALL=1 + export I_MPI_ADJUST_BCAST=4 + export I_MPI_ADJUST_IBCAST=1 + EOT + }, + ] + + rdma_runner = !var.install_cloud_rdma_drivers ? [] : [ + { + type = "shell" + source = "${path.module}/files/install_cloud_rdma_drivers.sh" + destination = "install_cloud_rdma_drivers.sh" + } + ] + + docker_runner = !var.docker.enabled ? [] : [ + { + type = "data" + destination = local.ansible_docker_settings_file + content = jsonencode({ + enable_docker_world_writable = var.docker.world_writable + docker_daemon_config = var.docker.daemon_config + docker_data_root = local.docker_data_root + }) + }, + { + type = "ansible-local" + destination = "install_docker.yml" + content = file("${path.module}/files/install_docker.yml") + args = "-e \"@${local.ansible_docker_settings_file}\"" + }, + ] + + managed_lustre_runner = !var.managed_lustre.enabled ? [] : [ + { + type = "ansible-local" + destination = "install_managed_lustre.yml" + content = file("${path.module}/files/install_managed_lustre.yml") + args = "-e managed_lustre_port=${var.managed_lustre.port}" + }, + ] + + gpu_network_wait_online_runner = !var.enable_gpu_network_wait_online ? [] : [ + { + type = "ansible-local" + destination = "install_gpu_network_wait_online.yml" + content = file("${path.module}/files/install_gpu_network_wait_online.yml") + args = "" + }, + ] + + local_ssd_filesystem_enabled = can(coalesce(var.local_ssd_filesystem.mountpoint)) + raid_setup = !local.local_ssd_filesystem_enabled ? [] : [ + { + type = "ansible-local" + destination = "setup-raid.yml" + content = file("${path.module}/files/setup-raid.yml") + args = join(" ", [ + "-e mountpoint=${var.local_ssd_filesystem.mountpoint}", + "-e fs_type=${var.local_ssd_filesystem.fs_type}", + "-e mode=${var.local_ssd_filesystem.permissions}", + ]) + }, + ] + + supplied_ansible_runners = anytrue([for r in var.runners : r.type == "ansible-local"]) + has_ansible_runners = anytrue([ + local.supplied_ansible_runners, + local.configure_ssh, + var.docker.enabled, + var.managed_lustre.enabled, + var.enable_gpu_network_wait_online, + local.local_ssd_filesystem_enabled + ]) + + install_ansible = coalesce(var.install_ansible, local.has_ansible_runners) + ansible_installer = local.install_ansible ? [{ + type = "shell" + source = "${path.module}/files/install_ansible.sh" + destination = "install_ansible_automatic.sh" + args = var.ansible_virtualenv_path + }] : [] + + hotfix_runner = [{ + type = "shell" + source = "${path.module}/files/early_run_hotfixes.sh" + destination = "early_run_hotfixes.sh" + }] + + runners = concat( + local.warnings, + local.hotfix_runner, + local.proxy_runner, + local.ofi_runner, + local.rdma_runner, + local.monitoring_agent_installer, + local.ansible_installer, + local.raid_setup, # order RAID early to ensure filesystem is ready for subsequent runners + local.managed_lustre_runner, + local.configure_ssh_runners, + local.docker_runner, + local.gpu_network_wait_online_runner, + var.runners + ) + + bucket_regex = "^gs://([^/]*)/*(.*)" + gcs_bucket_path_trimmed = var.gcs_bucket_path == null ? null : trimsuffix(var.gcs_bucket_path, "/") + storage_folder_path = local.gcs_bucket_path_trimmed == null ? null : regex(local.bucket_regex, local.gcs_bucket_path_trimmed)[1] + storage_folder_path_prefix = local.storage_folder_path == null || local.storage_folder_path == "" ? "" : "${local.storage_folder_path}/" + + user_provided_bucket_name = try(regex(local.bucket_regex, local.gcs_bucket_path_trimmed)[0], null) + storage_bucket_name = coalesce(one(google_storage_bucket.configs_bucket[*].name), local.user_provided_bucket_name) + + load_runners = templatefile( + "${path.module}/templates/startup-script-custom.tftpl", + { + bucket = local.storage_bucket_name, + http_proxy = var.http_proxy, + no_proxy = var.http_no_proxy, + runners = [ + for runner in local.runners : { + object = google_storage_bucket_object.scripts[basename(runner["destination"])].output_name + type = runner["type"] + destination = runner["destination"] + args = contains(keys(runner), "args") ? runner["args"] : "" + } + ] + } + ) + + stdlib_head = file("${path.module}/files/startup-script-stdlib-head.sh") + get_from_bucket = file("${path.module}/files/get_from_bucket.sh") + stdlib_body = file("${path.module}/files/startup-script-stdlib-body.sh") + + # List representing complete content, to be concatenated together. + stdlib_list = [ + local.stdlib_head, + local.get_from_bucket, + local.load_runners, + local.stdlib_body, + ] + + # Final content output to the user + stdlib = join("", local.stdlib_list) + + runners_map = { for runner in local.runners : + basename(runner["destination"]) => { + content = lookup(runner, "content", null) + source = lookup(runner, "source", null) + } + } +} + +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + +resource "google_storage_bucket" "configs_bucket" { + count = var.gcs_bucket_path == null ? 1 : 0 + project = var.project_id + name = "${var.deployment_name}-startup-scripts-${random_id.resource_name_suffix.hex}" + uniform_bucket_level_access = true + location = var.region + storage_class = "REGIONAL" + labels = local.labels +} + +resource "google_storage_bucket_iam_binding" "viewers" { + bucket = local.storage_bucket_name + role = "roles/storage.objectViewer" + members = var.bucket_viewers +} + +resource "google_storage_bucket_object" "scripts" { + # this writes all scripts exactly once into GCS + for_each = local.runners_map + name = "${local.storage_folder_path_prefix}${each.key}-${substr(try(md5(each.value.content), filemd5(each.value.source)), 0, 4)}" + content = each.value.content + source = each.value.source + bucket = local.storage_bucket_name + timeouts { + create = "10m" + update = "10m" + } + + lifecycle { + precondition { + condition = !(var.install_cloud_ops_agent && var.install_stackdriver_agent) + error_message = "Only one of var.install_stackdriver_agent or var.install_cloud_ops_agent can be set. Stackdriver is recommended for best performance." + } + } +} + +resource "local_file" "debug_file" { + for_each = toset(var.debug_file != null ? [var.debug_file] : []) + filename = var.debug_file + content = local.stdlib +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/metadata.yaml new file mode 100644 index 0000000000..2ada34471f --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - storage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/outputs.tf new file mode 100644 index 0000000000..6a15082814 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/outputs.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "startup_script" { + description = "script to load and run all runners, as a string value." + value = local.stdlib + depends_on = [ + google_storage_bucket_iam_binding.viewers + ] +} + +output "compute_startup_script" { + description = "script to load and run all runners, as a string value. Targets the inputs for the slurm controller." + value = local.stdlib + depends_on = [ + google_storage_bucket_iam_binding.viewers + ] +} + +output "controller_startup_script" { + description = "script to load and run all runners, as a string value. Targets the inputs for the slurm controller." + value = local.stdlib + depends_on = [ + google_storage_bucket_iam_binding.viewers + ] +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl new file mode 100644 index 0000000000..3c894b00b0 --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl @@ -0,0 +1,65 @@ + + +stdlib::run_playbook() { + if [ ! "$(which ansible-playbook)" ]; then + stdlib::error "ansible-playbook not found"\ + "Please install ansible before running ansible-local runners." + exit 1 + fi + ansible-playbook --connection=local --inventory=localhost, --limit localhost $1 $2 + ret_code=$? + return $${ret_code} +} + +stdlib::runner() { + + type=$1 + object=$2 + destination=$3 + tmpdir=$4 + args=$5 + + destpath="$(dirname $destination)" + filename="$(basename $destination)" + + if [ "$destpath" = "." ]; then + destpath=$tmpdir + fi + + stdlib::get_from_bucket -u "gs://${bucket}/$object" -d "$destpath" -f "$filename" + + stdlib::info "=== start executing runner: $object ===" + case "$1" in + ansible-local) stdlib::run_playbook "$destpath/$filename" "$args";; + shell) chmod u+x /$destpath/$filename && $destpath/$filename $args;; + esac + + exit_code=$? + stdlib::info "=== $object finished with exit_code=$exit_code ===" + if [ "$exit_code" -ne "0" ] ; then + stdlib::error "=== execution of $object failed, exiting ===" + stdlib::announce_runners_end "$exit_code" + exit $exit_code + fi +} + +stdlib::load_runners(){ + tmpdir="$(mktemp -d)" + + stdlib::debug "=== BEGIN Running runners ===" + stdlib::announce_runners_start + + %{if http_proxy != "" ~} + stdlib::info "=== Setting HTTP_PROXY,HTTPS_PROXY to ${http_proxy} ===" + export http_proxy=${http_proxy} + export https_proxy=${http_proxy} + export NO_PROXY=${no_proxy} + %{endif ~} + + %{for r in runners ~} + stdlib::runner "${r.type}" "${r.object}" "${r.destination}" $${tmpdir} "${r.args}" + %{endfor ~} + + stdlib::announce_runners_end "0" + stdlib::debug "=== END Running runners ===" +} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/variables.tf new file mode 100644 index 0000000000..7080085ece --- /dev/null +++ b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/variables.tf @@ -0,0 +1,298 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "deployment_name" { + description = "Name of the HPC deployment, used to name GCS bucket for startup scripts." + type = string +} + +variable "region" { + description = "The region to deploy to" + type = string +} + +variable "gcs_bucket_path" { + description = "The GCS path for storage bucket and the object, starting with `gs://`." + type = string + default = null +} + +variable "bucket_viewers" { + description = "Additional service accounts or groups, users, and domains to which to grant read-only access to startup-script bucket (leave unset if using default Compute Engine service account)" + type = list(string) + default = [] + + validation { + condition = alltrue([ + for u in var.bucket_viewers : length(regexall("^(allUsers$|allAuthenticatedUsers$|user:|group:|serviceAccount:|domain:)", u)) > 0 + ]) + error_message = "Bucket viewer members must begin with user/group/serviceAccount/domain following https://cloud.google.com/iam/docs/reference/rest/v1/Policy#Binding" + } +} + +variable "debug_file" { + description = "Path to an optional local to be written with 'startup_script'." + type = string + default = null +} + +variable "labels" { + description = "Labels for the created GCS bucket. Key-value pairs." + type = map(string) +} + +variable "runners" { + description = < 0 + error_message = "The POSIX permissions for the mountpoint must be represented as a 3 or 4-digit octal" + } + + default = { + fs_type = "ext4" + mountpoint = "" + permissions = "0755" + } + + nullable = false +} + +variable "install_cloud_ops_agent" { + description = "Warning: Consider using `install_stackdriver_agent` for better performance. Run Google Ops Agent installation script if set to true." + type = bool + default = false +} + +variable "install_stackdriver_agent" { + description = "Run Google Stackdriver Agent installation script if set to true. Preferred over ops agent for performance." + type = bool + default = false +} + +variable "install_ansible" { + description = "Run Ansible installation script if either set to true or unset and runner of type 'ansible-local' are used." + type = bool + default = null +} + +variable "configure_ssh_host_patterns" { + description = < Date: Fri, 18 Jul 2025 10:37:51 +0100 Subject: [PATCH 12/21] pre-commit tweaks --- .../artifacts/DO_NOT_MODIFY_THIS_DIRECTORY | 1 - .../.ghpc/artifacts/expanded_blueprint.yaml | 112 - vdi-test-scott/.gitignore | 48 - vdi-test-scott/instructions.txt | 23 - vdi-test-scott/primary/main.tf | 80 - .../embedded/community/modules/README.md | 7 - .../compute/htcondor-execute-point/README.md | 271 - .../htcondor-execute-point/compute_image.tf | 30 - .../files/htcondor_configure.yml | 74 - .../files/htcondor_configure_autoscaler.yml | 98 - .../compute/htcondor-execute-point/main.tf | 218 - .../htcondor-execute-point/metadata.yaml | 20 - .../compute/htcondor-execute-point/outputs.tf | 25 - .../templates/condor_config.tftpl | 31 - .../download-condor-config.ps1.tftpl | 34 - .../htcondor-execute-point/variables.tf | 265 - .../htcondor-execute-point/versions.tf | 34 - .../community/modules/compute/mig/README.md | 45 - .../community/modules/compute/mig/main.tf | 85 - .../modules/compute/mig/metadata.yaml | 21 - .../community/modules/compute/mig/outputs.tf | 18 - .../modules/compute/mig/variables.tf | 86 - .../community/modules/compute/mig/versions.tf | 27 - .../modules/compute/notebook/README.md | 112 - .../modules/compute/notebook/main.tf | 96 - .../modules/compute/notebook/metadata.yaml | 20 - .../modules/compute/notebook/variables.tf | 111 - .../modules/compute/notebook/versions.tf | 29 - .../README.md | 139 - .../main.tf | 121 - .../metadata.yaml | 20 - .../outputs.tf | 36 - .../source_image_logic.tf | 77 - .../variables.tf | 402 - .../versions.tf | 29 - .../README.md | 85 - .../schedmd-slurm-gcp-v6-nodeset-tpu/main.tf | 59 - .../metadata.yaml | 21 - .../outputs.tf | 39 - .../variables.tf | 171 - .../versions.tf | 23 - .../schedmd-slurm-gcp-v6-nodeset/README.md | 226 - .../schedmd-slurm-gcp-v6-nodeset/main.tf | 224 - .../metadata.yaml | 21 - .../schedmd-slurm-gcp-v6-nodeset/outputs.tf | 102 - .../source_image_logic.tf | 77 - .../schedmd-slurm-gcp-v6-nodeset/variables.tf | 635 - .../schedmd-slurm-gcp-v6-nodeset/versions.tf | 29 - .../schedmd-slurm-gcp-v6-partition/README.md | 105 - .../schedmd-slurm-gcp-v6-partition/main.tf | 42 - .../metadata.yaml | 20 - .../schedmd-slurm-gcp-v6-partition/outputs.tf | 59 - .../variables.tf | 310 - .../versions.tf | 23 - .../container/artifact-registry/README.md | 157 - .../container/artifact-registry/main.tf | 268 - .../container/artifact-registry/metadata.yaml | 21 - .../container/artifact-registry/outputs.tf | 18 - .../container/artifact-registry/validation.tf | 49 - .../container/artifact-registry/variables.tf | 122 - .../container/artifact-registry/versions.tf | 27 - .../database/bigquery-dataset/README.md | 76 - .../modules/database/bigquery-dataset/main.tf | 32 - .../database/bigquery-dataset/metadata.yaml | 19 - .../database/bigquery-dataset/outputs.tf | 20 - .../database/bigquery-dataset/variables.tf | 36 - .../database/bigquery-dataset/versions.tf | 29 - .../modules/database/bigquery-table/README.md | 87 - .../modules/database/bigquery-table/main.tf | 37 - .../database/bigquery-table/metadata.yaml | 19 - .../database/bigquery-table/outputs.tf | 28 - .../database/bigquery-table/variables.tf | 46 - .../database/bigquery-table/versions.tf | 29 - .../slurm-cloudsql-federation/README.md | 107 - .../slurm-cloudsql-federation/main.tf | 165 - .../slurm-cloudsql-federation/metadata.yaml | 21 - .../slurm-cloudsql-federation/outputs.tf | 27 - .../slurm-cloudsql-federation/variables.tf | 173 - .../slurm-cloudsql-federation/versions.tf | 36 - .../file-system/DDN-EXAScaler/README.md | 158 - .../modules/file-system/DDN-EXAScaler/main.tf | 72 - .../file-system/DDN-EXAScaler/metadata.yaml | 22 - .../file-system/DDN-EXAScaler/outputs.tf | 90 - .../file-system/DDN-EXAScaler/variables.tf | 502 - .../file-system/DDN-EXAScaler/versions.tf | 24 - .../modules/file-system/Intel-DAOS/README.md | 1 - .../cloud-storage-bucket/README.md | 170 - .../file-system/cloud-storage-bucket/main.tf | 126 - .../cloud-storage-bucket/metadata.yaml | 18 - .../cloud-storage-bucket/outputs.tf | 69 - .../scripts/install-gcs-fuse.sh | 44 - .../cloud-storage-bucket/scripts/mount.sh | 58 - .../cloud-storage-bucket/variables.tf | 254 - .../cloud-storage-bucket/versions.tf | 39 - .../modules/file-system/nfs-server/README.md | 152 - .../modules/file-system/nfs-server/main.tf | 131 - .../file-system/nfs-server/metadata.yaml | 19 - .../modules/file-system/nfs-server/outputs.tf | 53 - .../nfs-server/scripts/install-nfs-client.sh | 37 - .../scripts/install-nfs-server.sh.tpl | 35 - .../file-system/nfs-server/scripts/mount.sh | 58 - .../file-system/nfs-server/scripts/mount.yaml | 39 - .../file-system/nfs-server/variables.tf | 194 - .../file-system/nfs-server/versions.tf | 37 - .../FSI_MonteCarlo.ipynb | 125 - .../files/fsi-montecarlo-on-batch/README.md | 97 - .../fsi-montecarlo-on-batch/iteration.sh | 23 - .../files/fsi-montecarlo-on-batch/main.tf | 102 - .../fsi-montecarlo-on-batch/mc_run.tpl.py | 157 - .../fsi-montecarlo-on-batch/mc_run.tpl.yaml | 36 - .../fsi-montecarlo-on-batch/mc_run_reqs.txt | 9 - .../fsi-montecarlo-on-batch/metadata.yaml | 18 - .../fsi-montecarlo-on-batch/variables.tf | 51 - .../files/fsi-montecarlo-on-batch/versions.tf | 43 - .../internal/slurm-gcp/instance/README.md | 100 - .../internal/slurm-gcp/instance/main.tf | 126 - .../internal/slurm-gcp/instance/outputs.tf | 41 - .../internal/slurm-gcp/instance/variables.tf | 119 - .../internal/slurm-gcp/instance/versions.tf | 31 - .../slurm-gcp/instance_template/README.md | 86 - .../files/startup_sh_unlinted | 161 - .../slurm-gcp/instance_template/main.tf | 164 - .../slurm-gcp/instance_template/outputs.tf | 43 - .../slurm-gcp/instance_template/variables.tf | 431 - .../slurm-gcp/instance_template/versions.tf | 25 - .../internal_instance_template/README.md | 87 - .../internal_instance_template/main.tf | 227 - .../internal_instance_template/outputs.tf | 33 - .../internal_instance_template/variables.tf | 398 - .../internal_instance_template/versions.tf | 30 - .../internal/slurm-gcp/login/README.md | 52 - .../modules/internal/slurm-gcp/login/main.tf | 110 - .../internal/slurm-gcp/login/outputs.tf | 25 - .../internal/slurm-gcp/login/variables.tf | 188 - .../internal/slurm-gcp/login/versions.tf | 29 - .../internal/slurm-gcp/nodeset_tpu/README.md | 95 - .../internal/slurm-gcp/nodeset_tpu/main.tf | 121 - .../internal/slurm-gcp/nodeset_tpu/outputs.tf | 30 - .../slurm-gcp/nodeset_tpu/variables.tf | 158 - .../slurm-gcp/nodeset_tpu/versions.tf | 30 - .../dependencies-installer/README.md | 61 - .../helm_install/README.md | 64 - .../helm_install/main.tf | 75 - .../helm_install/metadata.yaml | 19 - .../helm_install/variables.tf | 212 - .../helm_install/versions.tf | 24 - .../kubernetes_manifest/README.md | 40 - .../kubernetes_manifest/main.tf | 104 - .../kubernetes_manifest/metadata.yaml | 19 - .../kubernetes_manifest/variables.tf | 69 - .../kubernetes_manifest/versions.tf | 24 - .../management/dependencies-installer/main.tf | 183 - .../dependencies-installer/metadata.yaml | 19 - .../dependencies-installer/providers.tf | 25 - .../dependencies-installer/variables.tf | 70 - .../dependencies-installer/versions.tf | 30 - .../network/private-service-access/README.md | 98 - .../network/private-service-access/main.tf | 42 - .../private-service-access/metadata.yaml | 20 - .../network/private-service-access/outputs.tf | 43 - .../private-service-access/variables.tf | 47 - .../private-service-access/versions.tf | 41 - .../modules/project/new-project/README.md | 128 - .../modules/project/service-account/README.md | 111 - .../modules/project/service-account/main.tf | 37 - .../project/service-account/metadata.yaml | 19 - .../project/service-account/outputs.tf | 36 - .../project/service-account/variables.tf | 113 - .../project/service-account/versions.tf | 22 - .../project/service-enablement/README.md | 70 - .../project/service-enablement/main.tf | 28 - .../project/service-enablement/metadata.yaml | 19 - .../project/service-enablement/variables.tf | 31 - .../project/service-enablement/versions.tf | 29 - .../modules/pubsub/bigquery-sub/README.md | 87 - .../modules/pubsub/bigquery-sub/main.tf | 57 - .../modules/pubsub/bigquery-sub/metadata.yaml | 19 - .../modules/pubsub/bigquery-sub/outputs.tf | 20 - .../modules/pubsub/bigquery-sub/variables.tf | 51 - .../modules/pubsub/bigquery-sub/versions.tf | 35 - .../community/modules/pubsub/topic/README.md | 82 - .../community/modules/pubsub/topic/main.tf | 48 - .../modules/pubsub/topic/metadata.yaml | 19 - .../community/modules/pubsub/topic/outputs.tf | 26 - .../modules/pubsub/topic/variables.tf | 74 - .../modules/pubsub/topic/versions.tf | 32 - .../chrome-remote-desktop/README.md | 113 - .../chrome-remote-desktop/main.tf | 111 - .../chrome-remote-desktop/metadata.yaml | 18 - .../chrome-remote-desktop/outputs.tf | 25 - .../scripts/configure-chrome-desktop.yml | 61 - .../scripts/configure-grid-drivers.yml | 163 - .../scripts/disable-sleep.yml | 39 - .../chrome-remote-desktop/variables.tf | 277 - .../chrome-remote-desktop/versions.tf | 19 - .../scheduler/htcondor-access-point/README.md | 187 - .../files/htcondor_configure.yml | 120 - .../scheduler/htcondor-access-point/main.tf | 338 - .../htcondor-access-point/metadata.yaml | 20 - .../htcondor-access-point/outputs.tf | 25 - .../templates/condor_config.tftpl | 70 - .../htcondor-access-point/variables.tf | 266 - .../htcondor-access-point/versions.tf | 37 - .../htcondor-central-manager/README.md | 159 - .../files/htcondor_configure.yml | 72 - .../htcondor-central-manager/main.tf | 226 - .../htcondor-central-manager/metadata.yaml | 20 - .../htcondor-central-manager/outputs.tf | 30 - .../templates/condor_config.tftpl | 31 - .../htcondor-central-manager/variables.tf | 192 - .../htcondor-central-manager/versions.tf | 33 - .../scheduler/htcondor-pool-secrets/README.md | 172 - .../files/htcondor_secrets.yml | 102 - .../scheduler/htcondor-pool-secrets/main.tf | 168 - .../htcondor-pool-secrets/metadata.yaml | 20 - .../htcondor-pool-secrets/outputs.tf | 50 - .../templates/fetch-idtoken.ps1.tftpl | 26 - .../htcondor-pool-secrets/variables.tf | 67 - .../htcondor-pool-secrets/versions.tf | 33 - .../htcondor-service-accounts/README.md | 128 - .../htcondor-service-accounts/main.tf | 51 - .../htcondor-service-accounts/metadata.yaml | 19 - .../htcondor-service-accounts/outputs.tf | 30 - .../htcondor-service-accounts/variables.tf | 56 - .../htcondor-service-accounts/versions.tf | 19 - .../scheduler/htcondor-setup/README.md | 118 - .../modules/scheduler/htcondor-setup/main.tf | 68 - .../scheduler/htcondor-setup/metadata.yaml | 21 - .../scheduler/htcondor-setup/outputs.tf | 27 - .../scheduler/htcondor-setup/variables.tf | 55 - .../scheduler/htcondor-setup/versions.tf | 19 - .../schedmd-slurm-gcp-v6-controller/README.md | 405 - .../controller.tf | 213 - .../etc/htc-slurm.conf.tpl | 65 - .../etc/htc-slurmdbd.conf.tpl | 34 - .../etc/long-prolog-slurm.conf.tpl | 71 - .../schedmd-slurm-gcp-v6-controller/login.tf | 50 - .../schedmd-slurm-gcp-v6-controller/main.tf | 35 - .../metadata.yaml | 21 - .../modules/cleanup_compute/README.md | 42 - .../modules/cleanup_compute/main.tf | 46 - .../scripts/cleanup_compute.sh | 92 - .../modules/cleanup_compute/variables.tf | 71 - .../modules/cleanup_compute/versions.tf | 27 - .../modules/cleanup_tpu/README.md | 79 - .../modules/cleanup_tpu/main.tf | 32 - .../cleanup_tpu/scripts/cleanup_tpu.sh | 63 - .../modules/cleanup_tpu/variables.tf | 60 - .../modules/cleanup_tpu/versions.tf | 27 - .../modules/slurm_files/README.md | 119 - .../modules/slurm_files/etc/cgroup.conf.tpl | 7 - .../modules/slurm_files/etc/slurm.conf.tpl | 67 - .../modules/slurm_files/etc/slurmdbd.conf.tpl | 31 - .../slurm_files/files/external_epilog.sh | 18 - .../slurm_files/files/external_prolog.sh | 18 - .../slurm_files/files/setup_external.sh | 115 - .../modules/slurm_files/main.tf | 331 - .../modules/slurm_files/outputs.tf | 45 - .../modules/slurm_files/scripts/conf.py | 641 - .../modules/slurm_files/scripts/file_cache.py | 80 - .../slurm_files/scripts/get_tpu_vmcount.py | 76 - .../slurm_files/scripts/job_submit.lua.tpl | 103 - .../modules/slurm_files/scripts/load_bq.py | 352 - .../slurm_files/scripts/local_pubsub.py | 196 - .../modules/slurm_files/scripts/mig_flex.py | 214 - .../slurm_files/scripts/requirements-dev.txt | 9 - .../slurm_files/scripts/requirements.txt | 18 - .../modules/slurm_files/scripts/resume.py | 664 - .../slurm_files/scripts/resume_wrapper.sh | 41 - .../modules/slurm_files/scripts/setup.py | 648 - .../scripts/setup_network_storage.py | 327 - .../modules/slurm_files/scripts/slurmsync.py | 661 - .../modules/slurm_files/scripts/sort_nodes.py | 171 - .../modules/slurm_files/scripts/suspend.py | 126 - .../slurm_files/scripts/suspend_wrapper.sh | 28 - .../slurm_files/scripts/tests/common.py | 115 - .../slurm_files/scripts/tests/test_conf.py | 192 - .../slurm_files/scripts/tests/test_resume.py | 175 - .../scripts/tests/test_topology.py | 215 - .../slurm_files/scripts/tests/test_util.py | 586 - .../slurm_files/scripts/tools/gpu-test | 129 - .../slurm_files/scripts/tools/task-epilog | 67 - .../slurm_files/scripts/tools/task-prolog | 70 - .../modules/slurm_files/scripts/tpu.py | 331 - .../modules/slurm_files/scripts/util.py | 2148 --- .../slurm_files/scripts/watch_delete_vm_op.py | 124 - .../modules/slurm_files/variables.tf | 500 - .../modules/slurm_files/versions.tf | 37 - .../outputs.tf | 57 - .../partition.tf | 172 - .../slurm_files.tf | 191 - .../source_image_logic.tf | 77 - .../variables.tf | 809 - .../variables_controller_instance.tf | 382 - .../versions.tf | 33 - .../schedmd-slurm-gcp-v6-login/README.md | 134 - .../schedmd-slurm-gcp-v6-login/main.tf | 108 - .../schedmd-slurm-gcp-v6-login/metadata.yaml | 21 - .../schedmd-slurm-gcp-v6-login/outputs.tf | 18 - .../source_image_logic.tf | 77 - .../schedmd-slurm-gcp-v6-login/variables.tf | 419 - .../schedmd-slurm-gcp-v6-login/versions.tf | 29 - .../scripts/htcondor-install/README.md | 149 - .../htcondor-install/files/autoscaler.py | 417 - .../install-htcondor-autoscaler-deps.yml | 46 - .../files/install-htcondor.yaml | 94 - .../modules/scripts/htcondor-install/main.tf | 51 - .../scripts/htcondor-install/metadata.yaml | 18 - .../scripts/htcondor-install/outputs.tf | 30 - .../templates/install-htcondor.ps1.tftpl | 59 - .../scripts/htcondor-install/variables.tf | 51 - .../scripts/htcondor-install/versions.tf | 19 - .../modules/scripts/ramble-execute/README.md | 116 - .../modules/scripts/ramble-execute/main.tf | 71 - .../scripts/ramble-execute/metadata.yaml | 18 - .../modules/scripts/ramble-execute/outputs.tf | 53 - .../templates/ramble_execute.yml.tpl | 59 - .../scripts/ramble-execute/variables.tf | 114 - .../scripts/ramble-execute/versions.tf | 25 - .../modules/scripts/ramble-setup/README.md | 128 - .../modules/scripts/ramble-setup/main.tf | 113 - .../scripts/ramble-setup/metadata.yaml | 18 - .../modules/scripts/ramble-setup/outputs.tf | 61 - .../scripts/install_ramble_deps.yml | 50 - .../install_ramble_python_deps.yml.tftpl | 28 - .../templates/ramble_setup.yml.tftpl | 157 - .../modules/scripts/ramble-setup/variables.tf | 97 - .../modules/scripts/ramble-setup/versions.tf | 30 - .../modules/scripts/spack-execute/README.md | 141 - .../modules/scripts/spack-execute/main.tf | 70 - .../scripts/spack-execute/metadata.yaml | 18 - .../modules/scripts/spack-execute/outputs.tf | 45 - .../templates/execute_commands.yml.tpl | 59 - .../scripts/spack-execute/variables.tf | 103 - .../modules/scripts/spack-execute/versions.tf | 25 - .../modules/scripts/spack-setup/README.md | 382 - .../modules/scripts/spack-setup/main.tf | 120 - .../modules/scripts/spack-setup/metadata.yaml | 19 - .../modules/scripts/spack-setup/outputs.tf | 56 - .../scripts/install_spack_deps.yml | 50 - .../templates/spack_setup.yml.tftpl | 157 - .../modules/scripts/spack-setup/variables.tf | 106 - .../modules/scripts/spack-setup/versions.tf | 30 - .../modules/scripts/vdi-setup/README.md | 187 - .../modules/scripts/vdi-setup/main.tf | 134 - .../modules/scripts/vdi-setup/outputs.tf | 44 - .../modules/scripts/vdi-setup/roles.tar.gz | Bin 8732 -> 0 bytes .../vdi-setup/roles/base_os/tasks/debian.yaml | 41 - .../roles/base_os/tasks/gpu_driver_setup.yaml | 69 - .../tasks/gpu_driver_setup/debian.yaml | 30 - .../base_os/tasks/gpu_driver_setup/rocky.yaml | 36 - .../tasks/gpu_driver_setup/ubuntu.yaml | 73 - .../vdi-setup/roles/base_os/tasks/main.yaml | 56 - .../vdi-setup/roles/base_os/tasks/rocky.yaml | 42 - .../vdi-setup/roles/base_os/tasks/ubuntu.yaml | 138 - .../roles/secret_manager/defaults/main.yaml | 16 - .../roles/secret_manager/tasks/main.yaml | 102 - .../tasks/user_secret_tasks.yaml | 127 - .../user_provision/tasks/local_users.yaml | 51 - .../roles/user_provision/tasks/main.yaml | 40 - .../roles/user_provision/tasks/os_login.yaml | 14 - .../roles/vdi_tool/tasks/guacamole.yaml | 212 - .../vdi-setup/roles/vdi_tool/tasks/main.yaml | 26 - .../roles/vdi_tool/tasks/nomachine.yaml | 14 - .../roles/vdi_tool/tasks/workspot.yaml | 14 - .../templates/connection_bootstrap.sql.j2 | 72 - .../vdi_tool/templates/user_bootstrap.sql.j2 | 22 - .../vdi-setup/roles/vnc/tasks/main.yaml | 23 - .../vdi-setup/roles/vnc/tasks/tigervnc.yaml | 69 - .../vdi-setup/roles/vnc/tasks/tightvnc.yaml | 14 - .../roles/vnc/tasks/vnc_user_config.yaml | 59 - .../vdi-setup/roles/vnc/templates/config.j2 | 2 - .../roles/vnc/templates/vncserver.users.j2 | 3 - .../vnc/templates/vncserver@.service.rocky.j2 | 15 - .../templates/vncserver@.service.ubuntu.j2 | 16 - .../vdi-setup/templates/install.yaml.tftpl | 12 - .../vdi-setup/templates/vars.yaml.tftpl | 23 - .../modules/scripts/vdi-setup/validation.tf | 53 - .../modules/scripts/vdi-setup/variables.tf | 91 - .../modules/scripts/vdi-setup/versions.tf | 29 - .../scripts/wait-for-startup/README.md | 87 - .../modules/scripts/wait-for-startup/main.tf | 47 - .../scripts/wait-for-startup/metadata.yaml | 19 - .../scripts/wait-for-startup/outputs.tf | 15 - .../scripts/wait-for-startup-status.sh | 115 - .../scripts/wait-for-startup/variables.tf | 54 - .../scripts/wait-for-startup/versions.tf | 29 - .../scripts/windows-startup-script/README.md | 109 - .../scripts/windows-startup-script/main.tf | 34 - .../windows-startup-script/metadata.yaml | 18 - .../scripts/windows-startup-script/outputs.tf | 20 - .../templates/install_gpu_driver.ps1.tftpl | 38 - .../templates/setx_http_proxy.ps1 | 5 - .../windows-startup-script/variables.tf | 54 - .../windows-startup-script/versions.tf | 23 - .../modules/embedded/modules/README.md | 550 - .../compute/gke-job-template/README.md | 130 - .../modules/compute/gke-job-template/main.tf | 168 - .../compute/gke-job-template/metadata.yaml | 18 - .../compute/gke-job-template/outputs.tf | 27 - .../templates/gke-job-base.yaml.tftpl | 120 - .../compute/gke-job-template/variables.tf | 187 - .../compute/gke-job-template/versions.tf | 28 - .../modules/compute/gke-node-pool/README.md | 375 - .../compute/gke-node-pool/disk_definitions.tf | 38 - .../sample-tcpx-workload-job.yaml | 50 - .../sample-tcpxo-workload-job.yaml | 70 - .../scripts/enable-tcpx-in-workload.py | 185 - .../scripts/enable-tcpxo-in-workload.py | 186 - .../compute/gke-node-pool/gpu_direct.tf | 87 - .../compute/gke-node-pool/guest_cpus.tf | 32 - .../modules/compute/gke-node-pool/main.tf | 440 - .../compute/gke-node-pool/metadata.yaml | 21 - .../modules/compute/gke-node-pool/outputs.tf | 112 - .../gke-node-pool/reservation_definitions.tf | 99 - .../gke-node-pool/threads_per_core_calc.tf | 42 - .../compute/gke-node-pool/variables.tf | 472 - .../modules/compute/gke-node-pool/versions.tf | 34 - .../modules/compute/resource-policy/README.md | 82 - .../modules/compute/resource-policy/main.tf | 48 - .../compute/resource-policy/metadata.yaml | 19 - .../compute/resource-policy/outputs.tf | 28 - .../compute/resource-policy/variables.tf | 55 - .../compute/resource-policy/versions.tf | 34 - .../modules/compute/vm-instance/README.md | 258 - .../compute/vm-instance/compute_image.tf | 30 - .../modules/compute/vm-instance/main.tf | 349 - .../modules/compute/vm-instance/metadata.yaml | 19 - .../modules/compute/vm-instance/outputs.tf | 50 - .../startup_from_network_storage.tf | 65 - .../vm-instance/threads_per_core_calc.tf | 42 - .../modules/compute/vm-instance/variables.tf | 452 - .../modules/compute/vm-instance/versions.tf | 41 - .../modules/file-system/filestore/README.md | 248 - .../modules/file-system/filestore/main.tf | 116 - .../file-system/filestore/metadata.yaml | 19 - .../modules/file-system/filestore/outputs.tf | 62 - .../filestore/scripts/install-nfs-client.sh | 37 - .../file-system/filestore/scripts/mount.sh | 58 - .../file-system/filestore/scripts/mount.yaml | 39 - .../file-system/filestore/variables.tf | 189 - .../modules/file-system/filestore/versions.tf | 36 - .../gke-persistent-volume/README.md | 166 - .../file-system/gke-persistent-volume/main.tf | 124 - .../gke-persistent-volume/metadata.yaml | 18 - .../gke-persistent-volume/outputs.tf | 26 - .../templates/filestore-pv.yaml.tftpl | 23 - .../templates/filestore-pvc.yaml.tftpl | 17 - .../templates/gcs-pv.yaml.tftpl | 24 - .../templates/gcs-pvc.yaml.tftpl | 17 - .../gke-persistent-volume/variables.tf | 62 - .../gke-persistent-volume/versions.tf | 34 - .../modules/file-system/gke-storage/README.md | 133 - .../modules/file-system/gke-storage/main.tf | 78 - .../file-system/gke-storage/metadata.yaml | 18 - .../file-system/gke-storage/outputs.tf | 27 - .../hyperdisk-balanced-pvc.yaml.tftpl | 15 - .../hyperdisk-extreme-pvc.yaml.tftpl | 15 - .../hyperdisk-throughput-pvc.yaml.tftpl | 15 - .../parallelstore-pvc.yaml.tftpl | 15 - .../hyperdisk-balanced-sc.yaml.tftpl | 25 - .../hyperdisk-extreme-sc.yaml.tftpl | 24 - .../hyperdisk-throughput-sc.yaml.tftpl | 24 - .../storage-class/parallelstore-sc.yaml.tftpl | 21 - .../file-system/gke-storage/variables.tf | 138 - .../file-system/gke-storage/versions.tf | 21 - .../file-system/managed-lustre/README.md | 289 - .../file-system/managed-lustre/main.tf | 104 - .../file-system/managed-lustre/metadata.yaml | 19 - .../file-system/managed-lustre/outputs.tf | 43 - .../scripts/install-managed-lustre-client.sh | 84 - .../managed-lustre/scripts/mount.sh | 58 - .../file-system/managed-lustre/variables.tf | 131 - .../file-system/managed-lustre/versions.tf | 36 - .../file-system/parallelstore/README.md | 196 - .../modules/file-system/parallelstore/main.tf | 74 - .../file-system/parallelstore/metadata.yaml | 19 - .../file-system/parallelstore/outputs.tf | 47 - .../scripts/install-daos-client.sh | 112 - .../templates/mount-daos.sh.tftpl | 110 - .../file-system/parallelstore/variables.tf | 137 - .../file-system/parallelstore/versions.tf | 36 - .../pre-existing-network-storage/README.md | 192 - .../metadata.yaml | 18 - .../pre-existing-network-storage/outputs.tf | 124 - .../scripts/install-daos-client.sh | 112 - .../scripts/install-gcs-fuse.sh | 44 - .../scripts/install-managed-lustre-client.sh | 84 - .../scripts/install-nfs-client.sh | 37 - .../scripts/mount.sh | 58 - .../ddn_exascaler_luster_client_install.tftpl | 50 - .../templates/mount-daos.sh.tftpl | 110 - .../pre-existing-network-storage/variables.tf | 67 - .../pre-existing-network-storage/versions.tf | 19 - .../modules/internal/gpu-definition/README.md | 47 - .../modules/internal/gpu-definition/main.tf | 96 - .../internal/network-attachment/README.md | 54 - .../internal/network-attachment/main.tf | 70 - .../internal/network-attachment/metadata.yaml | 19 - .../modules/internal/vpc_peering/README.md | 56 - .../modules/internal/vpc_peering/main.tf | 80 - .../internal/vpc_peering/metadata.yaml | 19 - .../management/kubectl-apply/README.md | 171 - .../kubectl-apply/helm_install/README.md | 64 - .../kubectl-apply/helm_install/main.tf | 79 - .../kubectl-apply/helm_install/metadata.yaml | 19 - .../kubectl-apply/helm_install/variables.tf | 212 - .../kubectl-apply/helm_install/versions.tf | 24 - .../kubectl-apply/kubectl/README.md | 57 - .../management/kubectl-apply/kubectl/main.tf | 65 - .../kubectl-apply/kubectl/metadata.yaml | 19 - .../kubectl-apply/kubectl/variables.tf | 45 - .../kubectl-apply/kubectl/versions.tf | 30 - .../modules/management/kubectl-apply/main.tf | 234 - .../manifests/jobset-v0.5.2.yaml | 8349 ---------- .../manifests/jobset-v0.7.2.yaml | 8918 ---------- .../manifests/jobset-v0.8.1.yaml | 9075 ---------- .../manifests/kueue-v0.10.0.yaml | 13185 --------------- .../manifests/kueue-v0.10.1.yaml | 13175 --------------- .../manifests/kueue-v0.11.4.yaml | 13466 --------------- .../manifests/kueue-v0.12.2.yaml | 13851 ---------------- .../management/kubectl-apply/metadata.yaml | 19 - .../management/kubectl-apply/providers.tf | 33 - .../management/kubectl-apply/variables.tf | 166 - .../management/kubectl-apply/versions.tf | 38 - .../modules/monitoring/dashboard/README.md | 86 - .../dashboard/dashboards/Empty.json.tpl | 17 - .../dashboard/dashboards/HPC.json.tpl | 595 - .../modules/monitoring/dashboard/main.tf | 35 - .../monitoring/dashboard/metadata.yaml | 19 - .../modules/monitoring/dashboard/outputs.tf | 23 - .../modules/monitoring/dashboard/variables.tf | 52 - .../modules/monitoring/dashboard/versions.tf | 29 - .../modules/network/firewall-rules/README.md | 107 - .../modules/network/firewall-rules/main.tf | 39 - .../network/firewall-rules/metadata.yaml | 19 - .../network/firewall-rules/variables.tf | 76 - .../network/firewall-rules/versions.tf | 29 - .../modules/network/gpu-rdma-vpc/README.md | 143 - .../modules/network/gpu-rdma-vpc/main.tf | 79 - .../network/gpu-rdma-vpc/metadata.yaml | 19 - .../modules/network/gpu-rdma-vpc/outputs.tf | 59 - .../modules/network/gpu-rdma-vpc/variables.tf | 164 - .../modules/network/gpu-rdma-vpc/versions.tf | 19 - .../modules/network/multivpc/README.md | 136 - .../embedded/modules/network/multivpc/main.tf | 78 - .../modules/network/multivpc/metadata.yaml | 19 - .../modules/network/multivpc/outputs.tf | 50 - .../modules/network/multivpc/variables.tf | 201 - .../modules/network/multivpc/versions.tf | 19 - .../network/pre-existing-subnetwork/README.md | 94 - .../network/pre-existing-subnetwork/main.tf | 38 - .../pre-existing-subnetwork/metadata.yaml | 21 - .../pre-existing-subnetwork/outputs.tf | 35 - .../pre-existing-subnetwork/variables.tf | 39 - .../pre-existing-subnetwork/versions.tf | 29 - .../network/pre-existing-vpc/README.md | 110 - .../modules/network/pre-existing-vpc/main.tf | 53 - .../network/pre-existing-vpc/metadata.yaml | 19 - .../network/pre-existing-vpc/outputs.tf | 50 - .../network/pre-existing-vpc/variables.tf | 37 - .../network/pre-existing-vpc/versions.tf | 29 - .../embedded/modules/network/vpc/README.md | 237 - .../embedded/modules/network/vpc/main.tf | 256 - .../modules/network/vpc/metadata.yaml | 19 - .../embedded/modules/network/vpc/outputs.tf | 68 - .../embedded/modules/network/vpc/variables.tf | 301 - .../embedded/modules/network/vpc/versions.tf | 19 - .../modules/packer/custom-image/README.md | 320 - .../modules/packer/custom-image/image.pkr.hcl | 216 - .../modules/packer/custom-image/metadata.yaml | 21 - .../packer/custom-image/variables.pkr.hcl | 276 - .../packer/custom-image/versions.pkr.hcl | 25 - .../scheduler/batch-job-template/README.md | 197 - .../batch-job-template/compute_image.tf | 30 - .../scheduler/batch-job-template/main.tf | 149 - .../batch-job-template/metadata.yaml | 22 - .../scheduler/batch-job-template/outputs.tf | 80 - .../startup_from_network_storage.tf | 65 - .../templates/batch-job-base.yaml.tftpl | 53 - .../templates/batch-submit.sh.tftpl | 10 - .../scheduler/batch-job-template/variables.tf | 240 - .../scheduler/batch-job-template/versions.tf | 37 - .../scheduler/batch-login-node/README.md | 127 - .../scheduler/batch-login-node/main.tf | 127 - .../scheduler/batch-login-node/metadata.yaml | 21 - .../scheduler/batch-login-node/outputs.tf | 37 - .../scheduler/batch-login-node/variables.tf | 151 - .../scheduler/batch-login-node/versions.tf | 29 - .../modules/scheduler/gke-cluster/README.md | 215 - .../modules/scheduler/gke-cluster/main.tf | 447 - .../scheduler/gke-cluster/metadata.yaml | 19 - .../modules/scheduler/gke-cluster/outputs.tf | 104 - .../templates/gke-network-paramset.yaml.tftpl | 9 - .../templates/network-object.yaml.tftpl | 11 - .../scheduler/gke-cluster/variables.tf | 521 - .../modules/scheduler/gke-cluster/versions.tf | 39 - .../pre-existing-gke-cluster/README.md | 116 - .../pre-existing-gke-cluster/main.tf | 70 - .../pre-existing-gke-cluster/metadata.yaml | 19 - .../pre-existing-gke-cluster/outputs.tf | 33 - .../templates/gke-network-paramset.yaml.tftpl | 9 - .../templates/network-object.yaml.tftpl | 11 - .../pre-existing-gke-cluster/variables.tf | 61 - .../pre-existing-gke-cluster/versions.tf | 30 - .../modules/scripts/startup-script/README.md | 348 - .../startup-script/files/configure-ssh.yml | 37 - .../startup-script/files/configure_proxy.sh | 54 - .../files/early_run_hotfixes.sh | 32 - .../startup-script/files/get_from_bucket.sh | 73 - .../startup-script/files/install_ansible.sh | 247 - .../files/install_cloud_rdma_drivers.sh | 38 - .../startup-script/files/install_docker.yml | 167 - .../files/install_gpu_network_wait_online.yml | 54 - .../files/install_managed_lustre.yml | 33 - .../files/install_monitoring_agent.sh | 144 - .../files/running-script-warning.sh | 26 - .../startup-script/files/setup-raid.yml | 100 - .../startup-script/files/setup-ssh-keys.sh | 19 - .../startup-script/files/setup-ssh-keys.yml | 40 - .../files/startup-script-stdlib-body.sh | 39 - .../files/startup-script-stdlib-head.sh | 266 - .../modules/scripts/startup-script/main.tf | 305 - .../scripts/startup-script/metadata.yaml | 19 - .../modules/scripts/startup-script/outputs.tf | 39 - .../templates/startup-script-custom.tftpl | 65 - .../scripts/startup-script/variables.tf | 298 - .../scripts/startup-script/versions.tf | 37 - vdi-test-scott/primary/providers.tf | 27 - vdi-test-scott/primary/variables.tf | 40 - vdi-test-scott/primary/versions.tf | 30 - 631 files changed, 140197 deletions(-) delete mode 100644 vdi-test-scott/.ghpc/artifacts/DO_NOT_MODIFY_THIS_DIRECTORY delete mode 100644 vdi-test-scott/.ghpc/artifacts/expanded_blueprint.yaml delete mode 100644 vdi-test-scott/.gitignore delete mode 100644 vdi-test-scott/instructions.txt delete mode 100644 vdi-test-scott/primary/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/validation.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/Intel-DAOS/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/scripts/mount.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/providers.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/new-project/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/partition.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables_controller_instance.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles.tar.gz delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/debian.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/debian.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/rocky.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/ubuntu.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/rocky.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/defaults/main.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/os_login.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/nomachine.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/workspot.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/connection_bootstrap.sql.j2 delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/user_bootstrap.sql.j2 delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/tigervnc.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/tightvnc.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/vnc_user_config.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/config.j2 delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver.users.j2 delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.rocky.j2 delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.ubuntu.j2 delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/validation.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpx-in-workload.py delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/compute_image.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/mount.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/mount.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-managed-lustre-client.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/templates/mount-daos.sh.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/internal/gpu-definition/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/internal/gpu-definition/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/jobset-v0.5.2.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/jobset-v0.7.2.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/jobset-v0.8.1.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.10.0.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.10.1.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.12.2.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/providers.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/multivpc/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/multivpc/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/multivpc/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/multivpc/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/multivpc/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/multivpc/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/vpc/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/vpc/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/vpc/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/vpc/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/vpc/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/network/vpc/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/image.pkr.hcl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/variables.pkr.hcl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/versions.pkr.hcl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/README.md delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_docker.yml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/main.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/metadata.yaml delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/outputs.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/variables.tf delete mode 100644 vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/versions.tf delete mode 100644 vdi-test-scott/primary/providers.tf delete mode 100644 vdi-test-scott/primary/variables.tf delete mode 100644 vdi-test-scott/primary/versions.tf diff --git a/vdi-test-scott/.ghpc/artifacts/DO_NOT_MODIFY_THIS_DIRECTORY b/vdi-test-scott/.ghpc/artifacts/DO_NOT_MODIFY_THIS_DIRECTORY deleted file mode 100644 index 56f49c329a..0000000000 --- a/vdi-test-scott/.ghpc/artifacts/DO_NOT_MODIFY_THIS_DIRECTORY +++ /dev/null @@ -1 +0,0 @@ -Files in this directory are managed by gcluster. Do not modify them manually! diff --git a/vdi-test-scott/.ghpc/artifacts/expanded_blueprint.yaml b/vdi-test-scott/.ghpc/artifacts/expanded_blueprint.yaml deleted file mode 100644 index eac3bcfc02..0000000000 --- a/vdi-test-scott/.ghpc/artifacts/expanded_blueprint.yaml +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -blueprint_name: vdi-test-scott -ghpc_version: v1.57.2-80-ge3e815e7a -vars: - deployment_name: vdi-test-scott - labels: - ghpc_blueprint: vdi-test-scott - ghpc_deployment: ((var.deployment_name)) - project_id: hpc-discovery-external - region: us-central1 - zone: us-central1-a -deployment_groups: - - group: primary - terraform_providers: - google: - source: hashicorp/google - version: ~> 6.41.0 - configuration: - project: ((var.project_id)) - region: ((var.region)) - zone: ((var.zone)) - google-beta: - source: hashicorp/google-beta - version: ~> 6.41.0 - configuration: - project: ((var.project_id)) - region: ((var.region)) - zone: ((var.zone)) - modules: - - source: modules/network/vpc - kind: terraform - id: network1 - settings: - deployment_name: ((var.deployment_name)) - extra_iap_ports: - - 8080 - firewall_rules: - - allow: - - ports: - - "8080" - protocol: tcp - description: Allow external ingress to Guacamole on TCP port 8080 - direction: INGRESS - name: allow-guacamole-8080-ext - ranges: - - 0.0.0.0/0 - labels: ((var.labels)) - project_id: ((var.project_id)) - region: ((var.region)) - - source: community/modules/project/service-enablement - kind: terraform - id: enable-apis - settings: - gcp_service_list: - - secretmanager.googleapis.com - - storage.googleapis.com - - compute.googleapis.com - project_id: ((var.project_id)) - - source: community/modules/scripts/vdi-setup - kind: terraform - id: vdi-setup - settings: - deployment_name: ((var.deployment_name)) - labels: ((var.labels)) - project_id: ((var.project_id)) - region: ((var.region)) - user_provision: local_users - vdi_resolution: 1920x1080 - vdi_tool: guacamole - vdi_user_group: vdiusers - vdi_users: - - port: 5901 - username: alice - - port: 5902 - secret_name: a-password-for-bob - username: bob - vnc_flavor: tigervnc - - source: modules/compute/vm-instance - kind: terraform - id: machine - use: - - network1 - - vdi-setup - settings: - deployment_name: ((var.deployment_name)) - disable_public_ips: false - instance_image: - family: debian-11 - project: debian-cloud - labels: ((var.labels)) - machine_type: g2-standard-8 - network_self_link: ((module.network1.network_self_link)) - project_id: ((var.project_id)) - region: ((var.region)) - startup_script: ((module.vdi-setup.startup_script)) - subnetwork_self_link: ((module.network1.subnetwork_self_link)) - tags: - - guacamole - zone: ((var.zone)) diff --git a/vdi-test-scott/.gitignore b/vdi-test-scott/.gitignore deleted file mode 100644 index 1e44b25074..0000000000 --- a/vdi-test-scott/.gitignore +++ /dev/null @@ -1,48 +0,0 @@ -# Local .terraform directories -**/.terraform/* - -# .tfstate files -*.tfstate -*.tfstate.* - -# Crash log files -crash.log -crash.*.log - -# Exclude all .tfvars files, which are likely to contain sensitive data, such as -# password, private keys, and other secrets. These should not be part of version -# control as they are data points which are potentially sensitive and subject -# to change depending on the environment. -*.tfvars -*.tfvars.json - -# Ignore override files as they are usually used to override resources locally and so -# are not checked in -override.tf -override.tf.json -*_override.tf -*_override.tf.json - -# Include override files you do wish to add to version control using negated pattern -# !example_override.tf - -# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan -# example: *tfplan* - -# Ignore CLI configuration files -.terraformrc -terraform.rc - -# Cache objects -packer_cache/ - -# https://www.packer.io/guides/hcl/variables -# Exclude all .pkrvars.hcl files, which are likely to contain sensitive data, -# such as password, private keys, and other secrets. These should not be part of -# version control as they are data points which are potentially sensitive and -# subject to change depending on the environment. -# -*.pkrvars.hcl - -# For built boxes -*.box diff --git a/vdi-test-scott/instructions.txt b/vdi-test-scott/instructions.txt deleted file mode 100644 index 4d37623497..0000000000 --- a/vdi-test-scott/instructions.txt +++ /dev/null @@ -1,23 +0,0 @@ -Advanced Deployment Instructions -================================ - -Terraform group 'primary' was successfully created in directory vdi-test-scott/primary -To deploy, run the following commands: - -terraform -chdir=vdi-test-scott/primary init -terraform -chdir=vdi-test-scott/primary validate -terraform -chdir=vdi-test-scott/primary apply - -Destroying infrastructure when no longer needed -=============================================== - -Automated ---------- - -gcluster destroy vdi-test-scott - -Advanced / Manual ------------------ -Infrastructure should be destroyed in reverse order of creation: - -terraform -chdir=vdi-test-scott/primary destroy diff --git a/vdi-test-scott/primary/main.tf b/vdi-test-scott/primary/main.tf deleted file mode 100644 index ff7414b420..0000000000 --- a/vdi-test-scott/primary/main.tf +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -module "network1" { - source = "./modules/embedded/modules/network/vpc" - deployment_name = var.deployment_name - extra_iap_ports = [8080] - firewall_rules = [{ - allow = [{ - ports = ["8080"] - protocol = "tcp" - }] - description = "Allow external ingress to Guacamole on TCP port 8080" - direction = "INGRESS" - name = "allow-guacamole-8080-ext" - ranges = ["0.0.0.0/0"] - }] - labels = var.labels - project_id = var.project_id - region = var.region -} - -module "enable-apis" { - source = "./modules/embedded/community/modules/project/service-enablement" - gcp_service_list = ["secretmanager.googleapis.com", "storage.googleapis.com", "compute.googleapis.com"] - project_id = var.project_id -} - -module "vdi-setup" { - source = "./modules/embedded/community/modules/scripts/vdi-setup" - deployment_name = var.deployment_name - labels = var.labels - project_id = var.project_id - region = var.region - user_provision = "local_users" - vdi_resolution = "1920x1080" - vdi_tool = "guacamole" - vdi_user_group = "vdiusers" - vdi_users = [{ - port = 5901 - username = "alice" - }, { - port = 5902 - secret_name = "a-password-for-bob" - username = "bob" - }] - vnc_flavor = "tigervnc" -} - -module "machine" { - source = "./modules/embedded/modules/compute/vm-instance" - deployment_name = var.deployment_name - disable_public_ips = false - instance_image = { - family = "debian-11" - project = "debian-cloud" - } - labels = var.labels - machine_type = "g2-standard-8" - network_self_link = module.network1.network_self_link - project_id = var.project_id - region = var.region - startup_script = module.vdi-setup.startup_script - subnetwork_self_link = module.network1.subnetwork_self_link - tags = ["guacamole"] - zone = var.zone -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/README.md deleted file mode 100644 index 0c83b9d30c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Community Modules - -This directory contains modules that rely on partner resources, have been -contributed by outside developers or are in early development by the Cluster Toolkit -team. The modules in this directory are listed alongside core modules in the -[core modules README](../../modules/README.md). There you can also learn more -about general use and how to write custom Cluster Toolkit modules. diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/README.md deleted file mode 100644 index 4f65411ddf..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/README.md +++ /dev/null @@ -1,271 +0,0 @@ -## Description - -This module performs the following tasks: - -- create an instance template from which execute points will be created -- create a managed instance group ([MIG][mig]) for execute points -- create a Toolkit runner to configure the autoscaler to scale the MIG - -It is expected to be used with the [htcondor-install] and [htcondor-setup] -modules. - -[htcondor-install]: ../../scripts/htcondor-install/README.md -[htcondor-setup]: ../../scheduler/htcondor-setup/README.md -[mig]: https://cloud.google.com/compute/docs/instance-groups/ - -### Known limitations - -This module may be used multiple times in a blueprint to create sets of -execute points in an HTCondor pool. If used more than 1 time, the setting -[name_prefix](#input_name_prefix) must be set to a value that is unique across -all uses of the htcondor-execute-point module. If you do not follow this -constraint, you will likely receive an error while running `terraform apply` -similar to that shown below. - -```text -Error: Invalid value for variable - - on modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf line 136, in module "startup_script": - 136: runners = local.all_runners - ā”œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ - ā”‚ var.runners is list of map of string with 5 elements - -All startup-script runners must have a unique destination. -``` - -### How to configure jobs to select execute points - -HTCondor access points provisioned by the Toolkit are specially configured to -honor an attribute named `RequireId` in each [Job ClassAd][jobad]. This value -must be set to the ID of a MIG created by an instance of this module. The -[htcondor-access-point] module includes a setting `var.default_mig_id` that will -set this value automatically to the MIG ID corresponding to the module's -execute points. If this setting is left unset each job must specify `+RequireId` -explicitly. In all cases, the default value can be overridden explicitly as shown -below: - -```text -universe = vanilla -executable = /bin/echo -arguments = "Hello, World!" -output = out.$(ClusterId).$(ProcId) -error = err.$(ClusterId).$(ProcId) -log = log.$(ClusterId).$(ProcId) -request_cpus = 1 -request_memory = 100MB -+RequireId = "htcondor-pool-ep-mig" -queue -``` - -[htcondor-access-point]: ../../scheduler/htcondor-access-point/README.md -[jobad]: https://htcondor.readthedocs.io/en/latest/users-manual/matchmaking-with-classads.html - -### Example - -A full example can be found in the [examples README][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- - -The following code snippet creates a pool with 2 sets of HTCondor execute -points, one using On-demand pricing and the other using Spot pricing. They use -a startup script and network created in previous steps. - -```yaml -- id: htcondor_execute_point - source: community/modules/compute/htcondor-execute-point - use: - - network1 - - htcondor_secrets - - htcondor_setup - - htcondor_cm - settings: - instance_image: - project: $(vars.project_id) - family: $(vars.new_image_family) - min_idle: 2 - -- id: htcondor_execute_point_spot - source: community/modules/compute/htcondor-execute-point - use: - - network1 - - htcondor_secrets - - htcondor_setup - - htcondor_cm - settings: - instance_image: - project: $(vars.project_id) - family: $(vars.new_image_family) - spot: true - -- id: htcondor_access - source: community/modules/scheduler/htcondor-access-point - use: - - network1 - - htcondor_secrets - - htcondor_setup - - htcondor_cm - - htcondor_execute_point - - htcondor_execute_point_spot - settings: - default_mig_id: $(htcondor_execute_point.mig_id) - enable_public_ips: true - instance_image: - project: $(vars.project_id) - family: $(vars.new_image_family) - outputs: - - access_point_ips - - access_point_name -``` - -## Support - -HTCondor is maintained by the [Center for High Throughput Computing][chtc] at -the University of Wisconsin-Madison. Support for HTCondor is available via: - -- [Discussion lists](https://htcondor.org/mail-lists/) -- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) -- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) - -[chtc]: https://chtc.cs.wisc.edu/ - -## Behavior of Managed Instance Group (MIG) - -Regional [MIGs][mig] are used to provision Execute Points. By default, VMs -will be provisioned in any of the zones available in that region, however, it -can be constrained to run in fewer zones (or a single zone) using -[var.zones](#input_zones). - -When the configuration of an Execute Point is changed, the MIG can be configured -to [replace the VM][replacement] using a "proactive" or "opportunistic" policy. -By default, the policy is set to opportunistic. In practice, this means that -Execute Points will _NOT_ be automatically replaced by Terraform when changes to -the instance template / HTCondor configuration are made. We recommend leaving -this at the default value as it will allow the HTCondor autoscaler to replace -VMs when they become idle without disrupting running jobs. - -However, if it is desired [var.update_policy](#input_update_policy) can be set -to "PROACTIVE" to enable automatic replacement. This will disrupt running jobs -and send them back to the queue. Alternatively, one can leave the setting at -the default value of "OPPORTUNISTIC" and update: - -- intentionally by issuing an update via Cloud Console or using gcloud (below) -- VMs becomes unhealthy or are otherwise automatically replaced (e.g. regular - Google Cloud maintenance) - -For example, to manually update all instances in a MIG: - -```text -gcloud compute instance-groups managed update-instances \ - <> --all-instances --region <> \ - --project <> --minimal-action replace -``` - -[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type - -## Known Issues - -When using OS Login with "external users" (outside of the Google Cloud -organization), then Docker universe jobs will fail and cause the Docker daemon -to crash. This stems from the use of POSIX user ids (uid) outside the range -supported by Docker. Please consider disabling OS Login if this atypical -situation applies. - -```yaml -vars: - # add setting below to existing deployment variables - enable_oslogin: DISABLE -``` - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.1 | -| [google](#requirement\_google) | >= 4.0 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.0 | -| [null](#provider\_null) | >= 3.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [execute\_point\_instance\_template](#module\_execute\_point\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | -| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | -| [mig](#module\_mig) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.execute_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [null_resource.execute_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | -| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [central\_manager\_ips](#input\_central\_manager\_ips) | List of IP addresses of HTCondor Central Managers | `list(string)` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `100` | no | -| [disk\_type](#input\_disk\_type) | Disk type for template | `string` | `"pd-balanced"` | no | -| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape across zones for instance group managing execute points | `string` | `"ANY"` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | -| [execute\_point\_runner](#input\_execute\_point\_runner) | A list of Toolkit runners for configuring an HTCondor execute point | `list(map(string))` | `[]` | no | -| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | Service account for HTCondor execute point (e-mail format) | `string` | n/a | yes | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | -| [instance\_image](#input\_instance\_image) | HTCondor execute point VM image

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | -| [labels](#input\_labels) | Labels to add to HTConodr execute points | `map(string)` | n/a | yes | -| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor execute points | `string` | `"n2-standard-4"` | no | -| [max\_size](#input\_max\_size) | Maximum size of the HTCondor execute point pool. | `number` | `5` | no | -| [metadata](#input\_metadata) | Metadata to add to HTCondor execute points | `map(string)` | `{}` | no | -| [min\_idle](#input\_min\_idle) | Minimum number of idle VMs in the HTCondor pool (if pool reaches var.max\_size, this minimum is not guaranteed); set to ensure jobs beginning run more quickly. | `number` | `0` | no | -| [name\_prefix](#input\_name\_prefix) | Name prefix given to hostnames in this group of execute points; must be unique across all instances of this module | `string` | n/a | yes | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network HTCondor execute points will join | `string` | `"default"` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | Project in which the HTCondor execute points will be created | `string` | n/a | yes | -| [region](#input\_region) | The region in which HTCondor execute points will be created | `string` | n/a | yes | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork HTCondor execute points will join | `string` | `null` | no | -| [target\_size](#input\_target\_size) | Initial size of the HTCondor execute point pool; set to null (default) to avoid Terraform management of size. | `number` | `null` | no | -| [update\_policy](#input\_update\_policy) | Replacement policy for Access Point Managed Instance Group ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle) | `string` | `"OPPORTUNISTIC"` | no | -| [windows\_startup\_ps1](#input\_windows\_startup\_ps1) | Startup script to run at boot-time for Windows-based HTCondor execute points | `list(string)` | `[]` | no | -| [zones](#input\_zones) | Zone(s) in which execute points may be created. If not supplied, will default to all zones in var.region. | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [autoscaler\_runner](#output\_autoscaler\_runner) | Toolkit runner to configure the HTCondor autoscaler | -| [mig\_id](#output\_mig\_id) | ID of the managed instance group containing the execute points | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf deleted file mode 100644 index 7a7fe02307..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/compute_image.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_compute_image" "compute_image" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = try(var.instance_image.project, null) - - lifecycle { - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml deleted file mode 100644 index 375ae036cd..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure.yml +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Configure HTCondor Role - hosts: localhost - become: true - vars: - spool_dir: /var/lib/condor/spool - condor_config_root: /etc/condor - ghpc_config_file: 50-ghpc-managed - tasks: - - name: Ensure necessary variables are set - ansible.builtin.assert: - that: - - htcondor_role is defined - - config_object is defined - - name: Remove default HTCondor configuration - ansible.builtin.file: - path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" - state: absent - notify: - - Reload HTCondor - - name: Create Toolkit configuration file - register: config_update - changed_when: config_update.rc == 137 - failed_when: config_update.rc != 0 and config_update.rc != 137 - ansible.builtin.shell: | - set -e -o pipefail - REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) - - CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" - if [ -f "${CONFIG_FILE}" ]; then - LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") - else - LOCAL_HASH="INVALID-HASH" - fi - - if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then - gcloud storage cp {{ config_object }} "${CONFIG_FILE}" - chmod 0644 "${CONFIG_FILE}" - exit 137 - fi - args: - executable: /bin/bash - notify: - - Reload HTCondor - handlers: - - name: Reload HTCondor - ansible.builtin.service: - name: condor - state: reloaded - post_tasks: - - name: Start HTCondor - ansible.builtin.service: - name: condor - state: started - enabled: true - - name: Inform users - changed_when: false - ansible.builtin.shell: | - set -e -o pipefail - wall "******* HTCondor system configuration complete ********" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml deleted file mode 100644 index a85158fdfc..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This playbook makes the assumption that a virtual environment has been created -# with the autoscaler and its dependencies previously installed. A runner that -# does this is provided as an output of the htcondor-install module within the -# Cluster Toolkit at community/modules/scripts/htcondor-install. - ---- -- name: Configure HTCondor Autoscaler - hosts: all - vars: - python: /usr/local/htcondor/bin/python3 - autoscaler: /usr/local/htcondor/bin/autoscaler.py - systemd_override_path: /etc/systemd/system - become: true - tasks: - - name: User must supply HTCondor role - ansible.builtin.assert: - that: - - project_id is defined - - region is defined - - zone is defined - - mig_id is defined - - max_size is defined - - name: Create SystemD service for HTCondor autoscaler - ansible.builtin.copy: - dest: "{{ systemd_override_path }}/htcondor-autoscaler@.service" - mode: 0644 - content: | - [Unit] - Description=HTCondor Autoscaler MIG: %i - - [Service] - User=condor - Type=oneshot - ExecStart={{ python }} {{ autoscaler }} --p $PROJECT_ID --r $REGION --z $ZONE --mz --g %i --c $MAX_SIZE --i $MIN_IDLE - notify: - - Reload SystemD - - name: Create SystemD override directory for autoscaler configuration - ansible.builtin.file: - path: "{{ systemd_override_path }}/htcondor-autoscaler@{{ mig_id }}.service.d" - state: directory - owner: root - group: root - mode: 0755 - - name: Create autoscaler configuration - ansible.builtin.copy: - dest: "{{ systemd_override_path }}/htcondor-autoscaler@{{ mig_id }}.service.d/miglimit.conf" - mode: 0644 - content: | - [Service] - Environment=PROJECT_ID={{ project_id }} - Environment=REGION={{ region }} - Environment=ZONE={{ zone }} - Environment=MAX_SIZE={{ max_size }} - Environment=MIN_IDLE={{ min_idle }} - notify: - - Reload SystemD - - name: Create SystemD timer for HTCondor autoscaler - ansible.builtin.copy: - dest: "{{ systemd_override_path }}/htcondor-autoscaler@.timer" - mode: 0644 - content: | - [Unit] - Description=Run HTCondor Autoscaler Periodically - - [Timer] - OnCalendar=minutely - AccuracySec=1us - RandomizedDelaySec=30 - # the directive below is ignored harmlessly on CentOS 7; this has impact - # that timing averages to 1 minute but is not precisely 1 minute; still - # useful to ensure that timers for different MIGs do not overlap - FixedRandomDelay=true - notify: - - Reload SystemD - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true - post_tasks: - - name: Activate HTCondor Autoscaler timer - ansible.builtin.systemd: - name: htcondor-autoscaler@{{ mig_id }}.timer - enabled: true - state: started diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf deleted file mode 100644 index 7b0df94987..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/main.tf +++ /dev/null @@ -1,218 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "htcondor-execute-point", ghpc_role = "compute" }) -} - -module "gpu" { - source = "../../../../modules/internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - zones = coalescelist(var.zones, data.google_compute_zones.available.names) - network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } - - oslogin_api_values = { - "DISABLE" = "FALSE" - "ENABLE" = "TRUE" - } - enable_oslogin = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } - - windows_startup_ps1 = join("\n\n", flatten([var.windows_startup_ps1, local.execute_config_windows_startup_ps1])) - - is_windows_image = anytrue([for l in data.google_compute_image.compute_image.licenses : length(regexall("windows-cloud", l)) > 0]) - windows_startup_metadata = local.is_windows_image && local.windows_startup_ps1 != "" ? { - windows-startup-script-ps1 = local.windows_startup_ps1 - } : {} - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - metadata = merge( - local.windows_startup_metadata, - local.network_storage_metadata, - local.enable_oslogin, - local.disable_automatic_updates_metadata, - var.metadata - ) - - autoscaler_runner = { - "type" = "ansible-local" - "content" = file("${path.module}/files/htcondor_configure_autoscaler.yml") - "destination" = "htcondor_configure_autoscaler_${module.mig.instance_group_manager.name}.yml" - "args" = join(" ", [ - "-e project_id=${var.project_id}", - "-e region=${var.region}", - "-e zone=${local.zones[0]}", # this value is required, but ignored by regional MIG autoscaler - "-e mig_id=${module.mig.instance_group_manager.name}", - "-e max_size=${var.max_size}", - "-e min_idle=${var.min_idle}", - ]) - } - - execute_config = templatefile("${path.module}/templates/condor_config.tftpl", { - htcondor_role = "get_htcondor_execute", - central_manager_ips = var.central_manager_ips, - guest_accelerator = local.guest_accelerator, - }) - - execute_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.execute_config.output_name}" - execute_runner = { - type = "ansible-local" - content = file("${path.module}/files/htcondor_configure.yml") - destination = "htcondor_configure.yml" - args = join(" ", [ - "-e htcondor_role=get_htcondor_execute", - "-e config_object=${local.execute_object}", - ]) - } - - native_fstype = [] - startup_script_network_storage = [ - for ns in var.network_storage : - ns if !contains(local.native_fstype, ns.fs_type) - ] - storage_client_install_runners = [ - for ns in local.startup_script_network_storage : - ns.client_install_runner if ns.client_install_runner != null - ] - mount_runners = [ - for ns in local.startup_script_network_storage : - ns.mount_runner if ns.mount_runner != null - ] - - all_runners = concat( - local.storage_client_install_runners, - local.mount_runners, - var.execute_point_runner, - [local.execute_runner], - ) - - execute_config_windows_startup_ps1 = templatefile( - "${path.module}/templates/download-condor-config.ps1.tftpl", - { - config_object = local.execute_object, - } - ) - - name_prefix = "${var.deployment_name}-${var.name_prefix}-ep" -} - -data "google_compute_zones" "available" { - project = var.project_id - region = var.region -} - -resource "null_resource" "execute_config" { - triggers = { - config = local.execute_config - } -} - -resource "google_storage_bucket_object" "execute_config" { - name = "${local.name_prefix}-config-${substr(md5(null_resource.execute_config.id), 0, 4)}" - content = local.execute_config - bucket = var.htcondor_bucket_name -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - project_id = var.project_id - region = var.region - labels = local.labels - deployment_name = var.deployment_name - - runners = local.all_runners -} - -module "execute_point_instance_template" { - source = "terraform-google-modules/vm/google//modules/instance_template" - version = "~> 12.1" - - name_prefix = local.name_prefix - project_id = var.project_id - network = var.network_self_link - subnetwork = var.subnetwork_self_link - service_account = { - email = var.execute_point_service_account_email - scopes = var.service_account_scopes - } - labels = local.labels - - machine_type = var.machine_type - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - gpu = one(local.guest_accelerator) - preemptible = var.spot - startup_script = local.is_windows_image ? null : module.startup_script.startup_script - metadata = local.metadata - source_image = data.google_compute_image.compute_image.self_link - - # secure boot - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config -} - -module "mig" { - source = "terraform-google-modules/vm/google//modules/mig" - version = "~> 12.1" - - project_id = var.project_id - region = var.region - distribution_policy_target_shape = var.distribution_policy_target_shape - distribution_policy_zones = local.zones - target_size = var.target_size - hostname = local.name_prefix - mig_name = local.name_prefix - instance_template = module.execute_point_instance_template.self_link - - health_check_name = "health-htcondor-${local.name_prefix}" - health_check = { - type = "tcp" - initial_delay_sec = 600 - check_interval_sec = 20 - healthy_threshold = 2 - timeout_sec = 8 - unhealthy_threshold = 3 - response = "" - proxy_header = "NONE" - port = 9618 - request = "" - request_path = "" - host = "" - enable_logging = true - } - - update_policy = [{ - instance_redistribution_type = "NONE" - replacement_method = "SUBSTITUTE" - max_surge_fixed = length(local.zones) - max_unavailable_fixed = length(local.zones) - max_surge_percent = null - max_unavailable_percent = null - min_ready_sec = 300 - minimal_action = "REPLACE" - type = var.update_policy - }] - -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml deleted file mode 100644 index 3a78f9a46b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - storage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf deleted file mode 100644 index b31f40130f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "autoscaler_runner" { - value = local.autoscaler_runner - description = "Toolkit runner to configure the HTCondor autoscaler" -} - -output "mig_id" { - value = module.mig.instance_group_manager.name - description = "ID of the managed instance group containing the execute points" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl deleted file mode 100644 index c8f5ce31a8..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/condor_config.tftpl +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# this file is managed by the Cluster Toolkit; do not edit it manually -# override settings with a higher priority (last lexically) named file -# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration - -use role:${htcondor_role} -CONDOR_HOST = ${join(",", central_manager_ips)} - -# StartD configuration settings -%{ if length(guest_accelerator) > 0 ~} -use feature:GPUs -%{ endif ~} -use feature:PartitionableSlot -use feature:CommonCloudAttributesGoogle("-c created-by") -UPDATE_INTERVAL = 30 -TRUST_UID_DOMAIN = True -STARTER_ALLOW_RUNAS_OWNER = True -RUNBENCHMARKS = False diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl deleted file mode 100644 index 19789f122e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/templates/download-condor-config.ps1.tftpl +++ /dev/null @@ -1,34 +0,0 @@ -# create directory for local condor_config customizations -$config_dir = 'C:\Condor\config' -if(!(test-path -PathType container -Path $config_dir)) -{ - New-Item -ItemType Directory -Path $config_dir -} - -# update local condor_config if blueprint has changed -$config_file = "$config_dir\50-ghpc-managed" -if (Test-Path -Path $config_file -PathType Leaf) -{ - $local_hash = gcloud --format="value(md5_hash)" storage hash $config_file -} -else -{ - $local_hash = "INVALID-HASH" -} - -$remote_hash = gcloud --format="value(md5_hash)" storage hash ${config_object} -if ($local_hash -cne $remote_hash) -{ - Write-Output "Updating condor configuration" - gcloud storage cp ${config_object} $config_file - if ($LASTEXITCODE -ne 0) - { - throw "Could not download HTCondor configuration; exiting startup script" - } - Restart-Service condor -} - -# ignored if service is already running; must be here to handle case where -# machine is rebooted, but configuration has previously been downloaded -# and service is disabled from automatic start -Start-Service condor diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf deleted file mode 100644 index aab8a54c2d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/variables.tf +++ /dev/null @@ -1,265 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HTCondor execute points will be created" - type = string -} - -variable "region" { - description = "The region in which HTCondor execute points will be created" - type = string -} - -variable "zones" { - description = "Zone(s) in which execute points may be created. If not supplied, will default to all zones in var.region." - type = list(string) - default = [] - nullable = false -} - -variable "distribution_policy_target_shape" { - description = "Target shape across zones for instance group managing execute points" - type = string - default = "ANY" -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "labels" { - description = "Labels to add to HTConodr execute points" - type = map(string) -} - -variable "machine_type" { - description = "Machine type to use for HTCondor execute points" - type = string - default = "n2-standard-4" -} - -variable "execute_point_runner" { - description = "A list of Toolkit runners for configuring an HTCondor execute point" - type = list(map(string)) - default = [] -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured" - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "instance_image" { - description = <<-EOD - HTCondor execute point VM image - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "cloud-hpc-image-public" - family = "hpc-rocky-linux-8" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "execute_point_service_account_email" { - description = "Service account for HTCondor execute point (e-mail format)" - type = string -} - -variable "service_account_scopes" { - description = "Scopes by which to limit service account attached to central manager." - type = set(string) - default = [ - "https://www.googleapis.com/auth/cloud-platform", - ] -} - -variable "network_self_link" { - description = "The self link of the network HTCondor execute points will join" - type = string - default = "default" -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork HTCondor execute points will join" - type = string - default = null -} - -variable "target_size" { - description = "Initial size of the HTCondor execute point pool; set to null (default) to avoid Terraform management of size." - type = number - default = null -} - -variable "max_size" { - description = "Maximum size of the HTCondor execute point pool." - type = number - default = 5 -} - -variable "min_idle" { - description = "Minimum number of idle VMs in the HTCondor pool (if pool reaches var.max_size, this minimum is not guaranteed); set to ensure jobs beginning run more quickly." - type = number - default = 0 -} - -variable "metadata" { - description = "Metadata to add to HTCondor execute points" - type = map(string) - default = {} -} - -# this default is deliberately the opposite of vm-instance because of observed -# issues running HTCondor docker universe jobs with OS Login enabled and running -# jobs as a user with uid>2^31; these uids occur when users outside the GCP -# organization login to a VM and OS Login is enabled. -variable "enable_oslogin" { - description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." - type = string - default = "ENABLE" - validation { - condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) - error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." - } -} - -variable "spot" { - description = "Provision VMs using discounted Spot pricing, allowing for preemption" - type = bool - default = false -} - -variable "disk_size_gb" { - description = "Boot disk size in GB" - type = number - default = 100 -} - -variable "disk_type" { - description = "Disk type for template" - type = string - default = "pd-balanced" -} - -variable "windows_startup_ps1" { - description = "Startup script to run at boot-time for Windows-based HTCondor execute points" - type = list(string) - default = [] - nullable = false -} - -variable "central_manager_ips" { - description = "List of IP addresses of HTCondor Central Managers" - type = list(string) -} - -variable "htcondor_bucket_name" { - description = "Name of HTCondor configuration bucket" - type = string -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = string, - count = number - })) - default = [] - nullable = false - - validation { - condition = length(var.guest_accelerator) <= 1 - error_message = "The HTCondor module supports 0 or 1 models of accelerator card on each execute point" - } -} - -variable "name_prefix" { - description = "Name prefix given to hostnames in this group of execute points; must be unique across all instances of this module" - type = string - nullable = false - validation { - condition = length(var.name_prefix) > 0 - error_message = "var.name_prefix must be a set to a non-empty string and must also be unique across all instances of htcondor-execute-point" - } -} - -variable "enable_shielded_vm" { - type = bool - default = false - description = "Enable the Shielded VM configuration (var.shielded_instance_config)." -} - -variable "shielded_instance_config" { - description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" - type = object({ - enable_secure_boot = bool - enable_vtpm = bool - enable_integrity_monitoring = bool - }) - - default = { - enable_secure_boot = true - enable_vtpm = true - enable_integrity_monitoring = true - } -} - -variable "update_policy" { - description = "Replacement policy for Access Point Managed Instance Group (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)" - type = string - default = "OPPORTUNISTIC" - validation { - condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) - error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf deleted file mode 100644 index 47c905b352..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/htcondor-execute-point/versions.tf +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 1.1" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.0" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } - } - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-execute-point/v1.57.0" - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/README.md deleted file mode 100644 index 278207b04a..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/README.md +++ /dev/null @@ -1,45 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | > 5.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | > 5.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_instance_group_manager.mig](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_group_manager) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [base\_instance\_name](#input\_base\_instance\_name) | Base name for the instances in the MIG | `string` | `null` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment, will be used to name MIG if `var.name` is not provided | `string` | n/a | yes | -| [ghpc\_module\_id](#input\_ghpc\_module\_id) | Internal GHPC field, do not set this value | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to the MIG | `map(string)` | n/a | yes | -| [name](#input\_name) | Name of the MIG. If not provided, will be generated from `var.deployment_name` | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the MIG will be created | `string` | n/a | yes | -| [target\_size](#input\_target\_size) | Target number of instances in the MIG | `number` | `0` | no | -| [versions](#input\_versions) | Application versions managed by this instance group. Each version deals with a specific instance template | 
list(object({
    name              = string
    instance_template = string
    target_size = optional(object({
      fixed   = optional(number)
      percent = optional(number)
    }))
  }))
| n/a | yes | -| [wait\_for\_instances](#input\_wait\_for\_instances) | Whether to wait for all instances to be created/updated before returning | `bool` | `false` | no | -| [zone](#input\_zone) | Compute Platform zone. Required, currently only zonal MIGs are supported | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [self\_link](#output\_self\_link) | The URL of the created MIG | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/main.tf deleted file mode 100644 index 0e7cf186c2..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/main.tf +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "mig", ghpc_role = "compute" }) -} - -locals { - sanitized_deploy_name = try(replace(lower(var.deployment_name), "/[^a-z0-9]/", ""), null) - sanitized_module_id = try(replace(lower(var.ghpc_module_id), "/[^a-z0-9]/", ""), null) - synth_mig_name = try("${local.sanitized_deploy_name}-${local.sanitized_module_id}", null) - - mig_name = var.name == null ? local.synth_mig_name : var.name - base_instance_name = var.base_instance_name == null ? local.mig_name : var.base_instance_name -} - -resource "google_compute_instance_group_manager" "mig" { - # REQUIRED - name = local.mig_name - base_instance_name = local.base_instance_name - zone = var.zone - - dynamic "version" { - for_each = var.versions - content { - name = version.value.name - instance_template = version.value.instance_template - dynamic "target_size" { - for_each = version.value.target_size != null ? [version.value.target_size] : [] - content { - fixed = target_size.value.fixed - percent = target_size.value.percent - } - } - } - } - - # OPTIONAL - project = var.project_id - target_size = var.target_size - wait_for_instances = var.wait_for_instances - - all_instances_config { - # TODO: validate that template metadata not getting wiped out - # TODO: validate that template labels not getting wiped out - labels = local.labels - } - - # OMITTED: - # * description - # * named_port - # * list_managed_instances_results - # * target_pools - specific for Load Balancers usage - # * wait_for_instances_status - # * auto_healing_policies - # * stateful_disk - # * stateful_internal_ip - # * update_policy - # * params - - - lifecycle { - precondition { - condition = local.mig_name != null - error_message = "Could not come up with a name for the MIG, specify `var.name`" - } - - precondition { - condition = local.base_instance_name != null - error_message = "Could not come up with a base_instance_name, specify `var.base_instance_name`" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/metadata.yaml deleted file mode 100644 index 97a4fa9a89..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com -ghpc: - inject_module_id: ghpc_module_id diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/outputs.tf deleted file mode 100644 index 23c66a3535..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/outputs.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "self_link" { - description = "The URL of the created MIG" - value = google_compute_instance_group_manager.mig.self_link -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/variables.tf deleted file mode 100644 index b6c3c0e78a..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/variables.tf +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - description = "Project in which the MIG will be created" - type = string -} - -variable "deployment_name" { - description = "Name of the deployment, will be used to name MIG if `var.name` is not provided" - type = string -} - -variable "labels" { - description = "Labels to add to the MIG" - type = map(string) -} - -variable "zone" { - description = "Compute Platform zone. Required, currently only zonal MIGs are supported" - type = string -} - - -variable "versions" { - description = <<-EOD - Application versions managed by this instance group. Each version deals with a specific instance template - EOD - type = list(object({ - name = string - instance_template = string - target_size = optional(object({ - fixed = optional(number) - percent = optional(number) - })) - })) - - validation { - condition = length(var.versions) > 0 - error_message = "At least one version must be provided" - } - -} - - -variable "ghpc_module_id" { - description = "Internal GHPC field, do not set this value" - type = string - default = null -} - -variable "name" { - description = "Name of the MIG. If not provided, will be generated from `var.deployment_name`" - type = string - default = null -} - -variable "base_instance_name" { - description = "Base name for the instances in the MIG" - type = string - default = null -} - - -variable "target_size" { - description = "Target number of instances in the MIG" - type = number - default = 0 -} - -variable "wait_for_instances" { - description = "Whether to wait for all instances to be created/updated before returning" - type = bool - default = false -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/versions.tf deleted file mode 100644 index a1045b9a6b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/mig/versions.tf +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.3" - - required_providers { - google = { - source = "hashicorp/google" - version = "> 5.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:mig/v1.57.0" - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/README.md deleted file mode 100644 index 506677ede8..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/README.md +++ /dev/null @@ -1,112 +0,0 @@ -# Description - -This module creates the Vertex AI Notebook, to be used in tutorials. - -Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -## Usage - -This is a simple usage, using the default network: - -```yaml - - id: bucket - source: community/modules/file-system/cloud-storage-bucket - settings: - name_prefix: my-bucket - local_mount: /home/jupyter/my-bucket - - - id: notebook - source: community/modules/compute/notebook - use: [bucket] - settings: - name_prefix: notebook - machine_type: n1-standard-4 - -``` - -If the user wants do specify a custom subnetwork, or specific external IP restrictions, they can use the `network_interfaces` variable, here is an example on how to use a Shared VPC Subnet with an ephemeral external IP: - -```yaml - - id: bucket - source: community/modules/file-system/cloud-storage-bucket - settings: - name_prefix: my-bucket - local_mount: /home/jupyter/my-bucket - - - id: notebook - source: community/modules/compute/notebook - use: [bucket] - settings: - name_prefix: notebook - machine_type: n1-standard-4 - network_interfaces: - - network: "projects/HOST_PROJECT_ID/global/networks/SHARED_VPC_NAME" - subnet: "projects/HOST_PROJECT_ID/regions/REGION/subnetworks/SUBNET_NAME" - nic_type: "VIRTIO_NET" -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [google](#requirement\_google) | >= 5.34 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 5.34 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.mount_script](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_workbench_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/workbench_instance) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment; used as part of name of the notebook. | `string` | n/a | yes | -| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | Bucket name, can be provided from the google-cloud-storage module | `string` | `null` | no | -| [instance\_image](#input\_instance\_image) | Instance Image | `map(string)` | 
{
  "family": "tf-latest-cpu",
  "name": null,
  "project": "deeplearning-platform-release"
}
| no | -| [labels](#input\_labels) | Labels to add to the resource Key-value pairs. | `map(string)` | n/a | yes | -| [machine\_type](#input\_machine\_type) | The machine type to employ | `string` | n/a | yes | -| [mount\_runner](#input\_mount\_runner) | mount content from the google-cloud-storage module | `map(string)` | n/a | yes | -| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces for the VM instance. Each network interface is represented by an object with the following fields:

- network: (Optional) The name of the Virtual Private Cloud (VPC) network that this VM instance is connected to.

- subnet: (Optional) The name of the subnetwork within the specified VPC that this VM instance is connected to.

- nic\_type: (Optional) The type of vNIC to be used on this interface. Possible values are: `VIRTIO_NET`, `GVNIC`.

- access\_configs: (Optional) An array of access configurations for this network interface. The access\_config object contains:
* external\_ip: (Required) An external IP address associated with this instance. Specify an unused static external IP address available to the project or leave this field undefined to use an IP from a shared ephemeral IP address pool. If you specify a static external IP address, it must live in the same region as the zone of the instance. | 
list(object({
    network  = optional(string)
    subnet   = optional(string)
    nic_type = optional(string)
    access_configs = optional(list(object({
      external_ip = optional(string)
    })))
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | ID of project in which the notebook will be created. | `string` | n/a | yes | -| [service\_account\_email](#input\_service\_account\_email) | If defined, the instance will use the service account specified instead of the Default Compute Engine Service Account | `string` | `null` | no | -| [zone](#input\_zone) | The zone to deploy to | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/main.tf deleted file mode 100644 index ad75526f50..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/main.tf +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "notebook", ghpc_role = "compute" }) -} - -locals { - suffix = random_id.resource_name_suffix.hex - #name = "thenotebook" - name = "notebook-${var.deployment_name}-${local.suffix}" - bucket = replace(var.gcs_bucket_path, "gs://", "") - post_script_filename = "mount-${local.suffix}.sh" - - # mount_runner_args is defined here: https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/3abddcfbd245b0e6747917a4e55b30658414ffd7/community/modules/file-system/cloud-storage-bucket/outputs.tf#L40 - mount_args = split(" ", var.mount_runner.args) - - unused = local.mount_args[0] - remote_mount = local.mount_args[1] - local_mount = local.mount_args[2] - fs_type = local.mount_args[3] - # These options provide a "rw" mount of the GCS bucket - mount_options = "defaults,_netdev,allow_other,implicit_dirs,gid=1000,uid=1000" - - content0 = var.mount_runner.content - content1 = replace(local.content0, "$1", local.unused) - content2 = replace(local.content1, "$2", local.remote_mount) - content3 = replace(local.content2, "$3", local.local_mount) - content4 = replace(local.content3, "$4", local.fs_type) - content5 = replace(local.content4, "$5", local.mount_options) - -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_storage_bucket_object" "mount_script" { - name = local.post_script_filename - content = local.content5 - bucket = local.bucket -} - -resource "google_workbench_instance" "instance" { - name = local.name - location = var.zone - project = var.project_id - labels = local.labels - gce_setup { - machine_type = var.machine_type - metadata = { - post-startup-script = "${var.gcs_bucket_path}/${google_storage_bucket_object.mount_script.name}" - } - vm_image { - project = var.instance_image.project - family = var.instance_image.family - } - - dynamic "service_accounts" { - for_each = var.service_account_email == null ? [] : [1] - content { - email = var.service_account_email - } - } - - dynamic "network_interfaces" { - for_each = var.network_interfaces - content { - network = network_interfaces.value.network - subnet = network_interfaces.value.subnet - nic_type = network_interfaces.value.nic_type - - dynamic "access_configs" { - for_each = network_interfaces.value.access_configs != null ? network_interfaces.value.access_configs : [] - content { - external_ip = access_configs.value.external_ip - } - } - } - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/metadata.yaml deleted file mode 100644 index 4a7d5397ca..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - notebooks.googleapis.com - - storage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/variables.tf deleted file mode 100644 index 4359de8c10..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/notebook/variables.tf +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which the notebook will be created." - type = string -} - -variable "deployment_name" { - description = "Name of the HPC deployment; used as part of name of the notebook." - type = string - # notebook name can have: lowercase letters, numbers, or hyphens (-) and cannot end with a hyphen - validation { - error_message = "The notebook name uses 'deployment_name' -- can only have: lowercase letters, numbers, or hyphens" - condition = can(regex("^[a-z0-9]+(?:-[a-z0-9]+)*$", var.deployment_name)) - } -} - -variable "zone" { - description = "The zone to deploy to" - type = string -} - -variable "machine_type" { - description = "The machine type to employ" - type = string -} - -variable "labels" { - description = "Labels to add to the resource Key-value pairs." - type = map(string) -} - -variable "instance_image" { - description = "Instance Image" - type = map(string) - default = { - project = "deeplearning-platform-release" - family = "tf-latest-cpu" - name = null - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "gcs_bucket_path" { - description = "Bucket name, can be provided from the google-cloud-storage module" - type = string - default = null -} - -variable "mount_runner" { - description = "mount content from the google-cloud-storage module" - type = map(string) - - validation { - condition = (length(split(" ", var.mount_runner.args)) == 5) - error_message = "There must be 5 elements in the Mount Runner Arguments: ${var.mount_runner.args} \n " - } -} - -variable "service_account_email" { - description = "If defined, the instance will use the service account specified instead of the Default Compute Engine Service Account" - type = string - default = null -} - -variable "network_interfaces" { - type = list(object({ - network = optional(string) - subnet = optional(string) - nic_type = optional(string) - access_configs = optional(list(object({ - external_ip = optional(string) - }))) - })) - default = [] - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 5.11 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 5.11 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | -| [slurm\_nodeset\_template](#module\_slurm\_nodeset\_template) | ../../internal/slurm-gcp/instance_template | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_compute_image.slurm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | -| [additional\_disks](#input\_additional\_disks) | Configurations of additional disks to be included on the partition nodes. | 
list(object({
    disk_name    = string
    device_name  = string
    disk_size_gb = number
    disk_type    = string
    disk_labels  = map(string)
    auto_delete  = bool
    boot         = bool
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | -| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | -| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of boot disk to create for the partition compute nodes. | `number` | `50` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-standard"` | no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | -| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [enable\_spot\_vm](#input\_enable\_spot\_vm) | Enable the partition to use spot VMs (https://cloud.google.com/spot-vms). | `bool` | `false` | no | -| [feature](#input\_feature) | The node feature, used to bind nodes to the nodeset. If not set, the nodeset name will be used. | `string` | `null` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm node group VM instances.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-9-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | -| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | -| [labels](#input\_labels) | Labels to add to partition compute instances. Key-value pairs. | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Compute Platform machine type to use for this partition compute nodes. | `string` | `"c2-standard-60"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | -| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy.

Note: Placement groups are not supported when on\_host\_maintenance is set to
"MIGRATE" and will be deactivated regardless of the value of
enable\_placement. To support enable\_placement, ensure on\_host\_maintenance is
set to "TERMINATE". | `string` | `"TERMINATE"` | no | -| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [region](#input\_region) | The default region for Cloud resources. | `string` | n/a | yes | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the compute instances. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the compute instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | Path to the Slurm bucket. | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster. | `string` | n/a | yes | -| [spot\_instance\_config](#input\_spot\_instance\_config) | Configuration for spot VMs. | 
object({
    termination_action = string
  })
| `null` | no | -| [startup\_script](#input\_startup\_script) | Startup script used by VMs in this nodeset | `string` | `"# no-op"` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | -| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instance\_template\_self\_link](#output\_instance\_template\_self\_link) | The URI of the template. | -| [node\_name\_prefix](#output\_node\_name\_prefix) | The prefix to be used for the node names.

Make sure that nodes are named `-`
This temporary required for proper functioning of the nodes.
While Slurm scheduler uses "features" to bind node and nodeset,
the SlurmGCP relies on node names for this (to be switched to features as well). | -| [nodeset\_dyn](#output\_nodeset\_dyn) | Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf deleted file mode 100644 index e52ddb972c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset-dynamic", ghpc_role = "compute" }) -} - -module "gpu" { - source = "../../../../modules/internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - nodeset_name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) - feature = coalesce(var.feature, local.nodeset_name) - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - universe_domain = { "universe_domain" = var.universe_domain } - - metadata = merge( - local.disable_automatic_updates_metadata, - local.universe_domain, - { slurmd_feature = local.feature }, - var.metadata - ) - - nodeset = { - nodeset_name = local.nodeset_name - nodeset_feature : local.feature - startup_script = local.ghpc_startup_script - network_storage = var.network_storage - } - - additional_disks = [ - for ad in var.additional_disks : { - disk_name = ad.disk_name - device_name = ad.device_name - disk_type = ad.disk_type - disk_size_gb = ad.disk_size_gb - disk_labels = merge(ad.disk_labels, local.labels) - auto_delete = ad.auto_delete - boot = ad.boot - } - ] - - public_access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] - access_config = length(var.access_config) == 0 ? local.public_access_config : var.access_config - - service_account = { - email = var.service_account_email - scopes = var.service_account_scopes - } - - ghpc_startup_script = [{ - filename = "ghpc_nodeset_startup.sh" - content = var.startup_script - }] - -} - -module "slurm_nodeset_template" { - source = "../../internal/slurm-gcp/instance_template" - - project_id = var.project_id - region = var.region - name_prefix = local.nodeset_name - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = "compute" - slurm_bucket_path = var.slurm_bucket_path - metadata = local.metadata - - additional_disks = local.additional_disks - disk_auto_delete = var.disk_auto_delete - disk_labels = merge(local.labels, var.disk_labels) - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - - bandwidth_tier = var.bandwidth_tier - can_ip_forward = var.can_ip_forward - - advanced_machine_features = var.advanced_machine_features - enable_confidential_vm = var.enable_confidential_vm - enable_oslogin = var.enable_oslogin - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config - - labels = local.labels - machine_type = var.machine_type - - min_cpu_platform = var.min_cpu_platform - on_host_maintenance = var.on_host_maintenance - termination_action = try(var.spot_instance_config.termination_action, null) - preemptible = var.preemptible - spot = var.enable_spot_vm - service_account = local.service_account - gpu = one(local.guest_accelerator) # requires gpu_definition.tf - source_image_family = local.source_image_family # requires source_image_logic.tf - source_image_project = local.source_image_project_normalized # requires source_image_logic.tf - source_image = local.source_image # requires source_image_logic.tf - - subnetwork = var.subnetwork_self_link - additional_networks = var.additional_networks - access_config = local.access_config - tags = concat([var.slurm_cluster_name], var.tags) -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml deleted file mode 100644 index a99e59d09f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [compute.googleapis.com] -ghpc: - inject_module_id: name diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf deleted file mode 100644 index 2d2d1415cf..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/outputs.tf +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "nodeset_dyn" { - description = "Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`." - value = local.nodeset -} - -output "instance_template_self_link" { - description = "The URI of the template." - value = module.slurm_nodeset_template.self_link -} - -output "node_name_prefix" { - description = <<-EOD - The prefix to be used for the node names. - - Make sure that nodes are named `-` - This temporary required for proper functioning of the nodes. - While Slurm scheduler uses "features" to bind node and nodeset, - the SlurmGCP relies on node names for this (to be switched to features as well). - EOD - value = "${var.slurm_cluster_name}-${local.nodeset_name}" - -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf deleted file mode 100644 index 0641cea7a8..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/source_image_logic.tf +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # Currently supported images and projects - known_project_families = { - schedmd-slurm-public = [ - "slurm-gcp-6-9-debian-11", - "slurm-gcp-6-9-hpc-rocky-linux-8", - "slurm-gcp-6-9-ubuntu-2004-lts", - "slurm-gcp-6-9-ubuntu-2204-lts-arm64" - ] - } - - # This approach to "hacking" the project name allows a chain of Terraform - # calls to set the instance source_image (boot disk) with a "relative - # resource name" that passes muster with VPC Service Control rules - # - # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 - # https://cloud.google.com/apis/design/resource_names#relative_resource_name - source_image_project_normalized = (can(var.instance_image.family) ? - "projects/${data.google_compute_image.slurm.project}/global/images/family" : - "projects/${data.google_compute_image.slurm.project}/global/images" - ) - source_image_family = can(var.instance_image.family) ? data.google_compute_image.slurm.family : "" - source_image = can(var.instance_image.name) ? data.google_compute_image.slurm.name : "" -} - -data "google_compute_image" "slurm" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = var.instance_image.project - - lifecycle { - precondition { - condition = length(regexall("^projects/.+?/global/images/family$", var.instance_image.project)) == 0 - error_message = "The \"project\" field in var.instance_image no longer supports a long-form ending in \"family\". Specify only the project ID." - } - - postcondition { - condition = var.instance_image_custom || contains(keys(local.known_project_families), self.project) - error_message = <<-EOD - Images in project ${self.project} are not published by SchedMD. Images must be created by compatible releases of the Terraform and Packer modules following the guidance at https://goo.gle/hpc-slurm-images. Set var.instance_image_custom to true to silence this error and acknowledge that you are using a compatible image. - EOD - } - postcondition { - condition = !contains(keys(local.known_project_families), self.project) || try(contains(local.known_project_families[self.project], self.family), false) - error_message = <<-EOD - Image family ${self.family} published by SchedMD in project ${self.project} is not compatible with this release of the Terraform Slurm modules. Select from known compatible releases: - ${join("\n", [for p in try(local.known_project_families[self.project], []) : "\t\"${p}\""])} - EOD - } - postcondition { - condition = var.disk_size_gb >= self.disk_size_gb - error_message = "'disk_size_gb: ${var.disk_size_gb}' is smaller than the image size (${self.disk_size_gb}GB), please increase the blueprint disk size" - } - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf deleted file mode 100644 index 595e424b81..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/variables.tf +++ /dev/null @@ -1,402 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "name" { - description = <<-EOD - Name of the nodeset. Automatically populated by the module id if not set. - If setting manually, ensure a unique value across all nodesets. - EOD - type = string -} - -variable "feature" { - type = string - description = "The node feature, used to bind nodes to the nodeset. If not set, the nodeset name will be used." - default = null -} - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "slurm_cluster_name" { - description = "Name of the Slurm cluster." - type = string -} - -variable "slurm_bucket_path" { - description = "Path to the Slurm bucket." - type = string -} - - -variable "machine_type" { - description = "Compute Platform machine type to use for this partition compute nodes." - type = string - default = "c2-standard-60" -} - -variable "metadata" { - type = map(string) - description = "Metadata, provided as a map." - default = {} -} - -variable "instance_image" { - description = <<-EOD - Defines the image that will be used in the Slurm node group VM instances. - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - - For more information on creating custom images that comply with Slurm on GCP - see the "Slurm on GCP Custom Images" section in docs/vm-images.md. - EOD - type = map(string) - default = { - family = "slurm-gcp-6-9-hpc-rocky-linux-8" - project = "schedmd-slurm-public" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "instance_image_custom" { - description = <<-EOD - A flag that designates that the user is aware that they are requesting - to use a custom and potentially incompatible image for this Slurm on - GCP module. - - If the field is set to false, only the compatible families and project - names will be accepted. The deployment will fail with any other image - family or name. If set to true, no checks will be done. - - See: https://goo.gle/hpc-slurm-images - EOD - type = bool - default = false -} - - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "tags" { - type = list(string) - description = "Network tag list." - default = [] -} - -variable "disk_type" { - description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." - type = string - default = "pd-standard" -} - -variable "disk_size_gb" { - description = "Size of boot disk to create for the partition compute nodes." - type = number - default = 50 -} - -variable "disk_auto_delete" { - type = bool - description = "Whether or not the boot disk should be auto-deleted." - default = true -} - -variable "disk_labels" { - description = "Labels specific to the boot disk. These will be merged with var.labels." - type = map(string) - default = {} -} - -variable "additional_disks" { - description = "Configurations of additional disks to be included on the partition nodes." - type = list(object({ - disk_name = string - device_name = string - disk_size_gb = number - disk_type = string - disk_labels = map(string) - auto_delete = bool - boot = bool - })) - default = [] -} - -variable "enable_confidential_vm" { - type = bool - description = "Enable the Confidential VM configuration. Note: the instance image must support option." - default = false -} - -variable "enable_shielded_vm" { - type = bool - description = "Enable the Shielded VM configuration. Note: the instance image must support option." - default = false -} - -variable "shielded_instance_config" { - type = object({ - enable_integrity_monitoring = bool - enable_secure_boot = bool - enable_vtpm = bool - }) - description = <<-EOD - Shielded VM configuration for the instance. Note: not used unless - enable_shielded_vm is 'true'. - - enable_integrity_monitoring : Compare the most recent boot measurements to the - integrity policy baseline and return a pair of pass/fail results depending on - whether they match or not. - - enable_secure_boot : Verify the digital signature of all boot components, and - halt the boot process if signature verification fails. - - enable_vtpm : Use a virtualized trusted platform module, which is a - specialized computer chip you can use to encrypt objects like keys and - certificates. - EOD - default = { - enable_integrity_monitoring = true - enable_secure_boot = true - enable_vtpm = true - } -} - - -variable "enable_oslogin" { - type = bool - description = <<-EOD - Enables Google Cloud os-login for user login and authentication for VMs. - See https://cloud.google.com/compute/docs/oslogin - EOD - default = true -} - -variable "can_ip_forward" { - description = "Enable IP forwarding, for NAT instances for example." - type = bool - default = false -} - -variable "advanced_machine_features" { - description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" - type = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) - default = { - threads_per_core = 1 # disable SMT by default - } -} - -variable "enable_smt" { # tflint-ignore: terraform_unused_declarations - type = bool - description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - default = null - validation { - condition = var.enable_smt == null - error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - } -} - -variable "labels" { - description = "Labels to add to partition compute instances. Key-value pairs." - type = map(string) - default = {} -} - -variable "min_cpu_platform" { - description = "The name of the minimum CPU platform that you want the instance to use." - type = string - default = null -} - -variable "on_host_maintenance" { - type = string - description = <<-EOD - Instance availability Policy. - - Note: Placement groups are not supported when on_host_maintenance is set to - "MIGRATE" and will be deactivated regardless of the value of - enable_placement. To support enable_placement, ensure on_host_maintenance is - set to "TERMINATE". - EOD - default = "TERMINATE" -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = string, - count = number - })) - default = [] - nullable = false - - validation { - condition = length(var.guest_accelerator) <= 1 - error_message = "The Slurm modules supports 0 or 1 models of accelerator card on each node." - } -} - -variable "preemptible" { - description = "Should use preemptibles to burst." - type = bool - default = false -} - - -variable "service_account_email" { - description = "Service account e-mail address to attach to the compute instances." - type = string - default = null -} - -variable "service_account_scopes" { - description = "Scopes to attach to the compute instances." - type = set(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -variable "enable_spot_vm" { - description = "Enable the partition to use spot VMs (https://cloud.google.com/spot-vms)." - type = bool - default = false -} - -variable "spot_instance_config" { - description = "Configuration for spot VMs." - type = object({ - termination_action = string - }) - default = null -} - -variable "bandwidth_tier" { - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [accelerator\_config](#input\_accelerator\_config) | Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details. | 
object({
    topology = string
    version  = string
  })
| 
{
  "topology": "",
  "version": ""
}
| no | -| [data\_disks](#input\_data\_disks) | The data disks to include in the TPU node | `list(string)` | `[]` | no | -| [disable\_public\_ips](#input\_disable\_public\_ips) | DEPRECATED: Use `enable_public_ips` instead. | `bool` | `null` | no | -| [docker\_image](#input\_docker\_image) | The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf- | `string` | `null` | no | -| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | -| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | -| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of auto-scaling worker nodes allowed in this partition.
For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores).
See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. | `number` | `0` | no | -| [node\_count\_static](#input\_node\_count\_static) | Number of worker nodes to be statically created.
For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores).
See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. | `number` | `0` | no | -| [node\_type](#input\_node\_type) | Specify a node type to base the vm configuration upon it. | `string` | `""` | no | -| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | -| [preserve\_tpu](#input\_preserve\_tpu) | Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [reserved](#input\_reserved) | Specify whether TPU-vms in this nodeset are created under a reservation. | `bool` | `false` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the TPU-vm. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the TPU-vm. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The name of the subnetwork to attach the TPU-vm of this nodeset to. | `string` | n/a | yes | -| [tf\_version](#input\_tf\_version) | Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details. | `string` | `"2.14.0"` | no | -| [zone](#input\_zone) | Zone in which to create compute VMs. TPU partitions can only specify a single zone. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [nodeset\_tpu](#output\_nodeset\_tpu) | Details of the nodeset tpu. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf deleted file mode 100644 index ac9b119702..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/main.tf +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# locals { -# # This label allows for billing report tracking based on module. -# labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset", ghpc_role = "compute" }) -# } - -locals { - name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) - - service_account = { - email = var.service_account_email - scopes = var.service_account_scopes - } - - nodeset_tpu = { - node_count_static = var.node_count_static - node_count_dynamic_max = var.node_count_dynamic_max - nodeset_name = local.name - node_type = var.node_type - - accelerator_config = var.accelerator_config - tf_version = var.tf_version - preemptible = var.preemptible - preserve_tpu = var.preserve_tpu - - data_disks = var.data_disks - docker_image = var.docker_image - - enable_public_ip = var.enable_public_ips - # TODO: rename to subnetwork_self_link, requires changes to the scripts - subnetwork = var.subnetwork_self_link - service_account = local.service_account - zone = var.zone - - project_id = var.project_id - reserved = var.reserved - network_storage = var.network_storage - } - - node_type_core_count = var.node_type == "" ? 0 : tonumber(regex("-(.*)", var.node_type)[0]) - - accelerator_core_list = var.accelerator_config.topology == "" ? [0, 0] : regexall("\\d+", var.accelerator_config.topology) - accelerator_core_count = length(local.accelerator_core_list) > 2 ? (local.accelerator_core_list[0] * local.accelerator_core_list[1] * local.accelerator_core_list[2]) * 2 : (local.accelerator_core_list[0] * local.accelerator_core_list[1]) * 2 - - tpu_core_count = local.accelerator_core_count == 0 ? local.node_type_core_count : local.accelerator_core_count -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml deleted file mode 100644 index 95b6d1c730..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] -ghpc: - inject_module_id: name - has_to_be_used: true diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf deleted file mode 100644 index 8cb7b8663e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/outputs.tf +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "nodeset_tpu" { - description = "Details of the nodeset tpu. Typically used as input to `schedmd-slurm-gcp-v6-partition`." - value = local.nodeset_tpu - - precondition { - condition = (var.node_type == "") != (var.accelerator_config == { topology : "", version : "" }) - error_message = "Either a node_type or an accelerator_config must be provided." - } - - precondition { - condition = ((local.tpu_core_count / 8) <= var.node_count_dynamic_max) || ((local.tpu_core_count / 8) <= var.node_count_static) - error_message = <<-EOD - When using TPUs there should be at least one node per every 8 cores. - Currently there are ${local.tpu_core_count} cores but only ${var.node_count_static} static nodes and ${var.node_count_dynamic_max} dynamic nodes. - EOD - } - - precondition { - condition = (var.node_count_dynamic_max % (local.tpu_core_count / 8) == 0) && (var.node_count_static % (local.tpu_core_count / 8) == 0) - error_message = <<-EOD - The number of worker nodes should be a multiple of ${local.tpu_core_count / 8}. - This is to ensure each node has a TPU machine for job scheduling. - EOD - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf deleted file mode 100644 index 367b0bee09..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/variables.tf +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "node_count_static" { - description = <<-EOD - Number of worker nodes to be statically created. - For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores). - See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. - EOD - type = number - default = 0 -} - -variable "node_count_dynamic_max" { - description = <<-EOD - Maximum number of auto-scaling worker nodes allowed in this partition. - For larger TPU machines, there are multiple worker nodes required per machine (1 for every 8 cores). - See https://cloud.google.com/tpu/docs/v4#large-topologies, for more information about these machine types. - EOD - type = number - default = 0 -} - -variable "name" { - description = <<-EOD - Name of the nodeset. Automatically populated by the module id if not set. - If setting manually, ensure a unique value across all nodesets. - EOD - type = string -} - -variable "enable_public_ips" { - description = "If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access_config is set." - type = bool - default = false -} - -variable "disable_public_ips" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: Use `enable_public_ips` instead." - type = bool - default = null - validation { - condition = var.disable_public_ips == null - error_message = "DEPRECATED: Use `enable_public_ips` instead." - } -} - -variable "node_type" { - description = "Specify a node type to base the vm configuration upon it." - type = string - default = "" -} - -variable "accelerator_config" { - description = "Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details." - type = object({ - topology = string - version = string - }) - default = { - topology = "" - version = "" - } - validation { - condition = var.accelerator_config.version == "" ? true : contains(["V2", "V3", "V4"], var.accelerator_config.version) - error_message = "accelerator_config.version must be one of [\"V2\", \"V3\", \"V4\"]" - } - validation { - condition = var.accelerator_config.topology == "" ? true : can(regex("^[1-9]x[1-9](x[1-9])?$", var.accelerator_config.topology)) - error_message = "accelerator_config.topology must be a valid topology, like 2x2 4x4x4 4x2x4 etc..." - } -} - -variable "tf_version" { - description = "Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details." - type = string - default = "2.14.0" -} - -variable "preemptible" { - description = "Should use preemptibles to burst." - type = bool - default = false -} - -variable "preserve_tpu" { - description = "Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted" - type = bool - default = false -} - -variable "zone" { - description = "Zone in which to create compute VMs. TPU partitions can only specify a single zone." - type = string -} - -variable "data_disks" { - description = "The data disks to include in the TPU node" - type = list(string) - default = [] -} - -variable "docker_image" { - description = "The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf-" - type = string - default = null -} - -variable "subnetwork_self_link" { - type = string - description = "The name of the subnetwork to attach the TPU-vm of this nodeset to." -} - -variable "service_account_email" { - description = "Service account e-mail address to attach to the TPU-vm." - type = string - default = null -} - -variable "service_account_scopes" { - description = "Scopes to attach to the TPU-vm." - type = set(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -variable "service_account" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." - type = object({ - email = string - scopes = set(string) - }) - default = null - validation { - condition = var.service_account == null - error_message = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." - } -} - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "reserved" { - description = "Specify whether TPU-vms in this nodeset are created under a reservation." - type = bool - default = false -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured on nodes." - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - })) - default = [] -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf deleted file mode 100644 index 8b9d422d23..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/versions.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.3" - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset-tpu/v1.57.0" - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md deleted file mode 100644 index bb742e72d1..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md +++ /dev/null @@ -1,226 +0,0 @@ -## Description - -This module creates a nodeset data structure intended to be input to the -[schedmd-slurm-gcp-v6-partition](../schedmd-slurm-gcp-v6-partition/) module. - -Nodesets allow adding heterogeneous node types to a partition, and hence -running jobs that mix multiple node characteristics. See the [heterogeneous jobs -section][hetjobs] of the SchedMD documentation for more information. - -To specify nodes from a specific nodesets in a partition, the [`--nodelist`] -(or `-w`) flag can be used, for example: - -```bash -srun -N 3 -p compute --nodelist cluster-compute-group-[0-2] hostname -``` - -Where the 3 nodes will be selected from the nodes `cluster-compute-group-[0-2]` -in the compute partition. - -Additionally, depending on how the nodes differ, a constraint can be added via -the [`--constraint`] (or `-C`) flag or other flags such as `--mincpus` can be -used to specify nodes with the desired characteristics. - -[`--nodelist`]: https://slurm.schedmd.com/srun.html#OPT_nodelist -[`--constraint`]: https://slurm.schedmd.com/srun.html#OPT_constraint -[hetjobs]: https://slurm.schedmd.com/heterogeneous_jobs.html - -### Example - -The following code snippet creates a partition module using the `nodeset` -module as input with: - -* a max node count of 200 -* VM machine type of `c2-standard-30` -* partition name of "compute" -* default nodeset name of "ghpc" -* connected to the `network` module via `use` -* nodes mounted to homefs via `use` - -```yaml -- id: nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: - - network - settings: - node_count_dynamic_max: 200 - machine_type: c2-standard-30 - -- id: compute_partition - source: community/modules/compute/schedmd-slurm-gcp-v6-partition - use: - - homefs - - nodeset - settings: - partition_name: compute -``` - -## Custom Images - -For more information on creating valid custom images for the node group VM -instances or for custom instance templates, see our [vm-images.md] documentation -page. - -[vm-images.md]: ../../../../docs/vm-images.md#slurm-on-gcp-custom-images - -## GPU Support - -More information on GPU support in Slurm on GCP and other Cluster Toolkit modules -can be found at [docs/gpu-support.md](../../../../docs/gpu-support.md) - -### Compute VM Zone Policies - -The Slurm on GCP nodeset module allows you to specify additional zones in -which to create VMs through [bulk creation][bulk]. This is valuable when -configuring partitions with popular VM families and you desire access to -more compute resources across zones. - -[bulk]: https://cloud.google.com/compute/docs/instances/multiple/about-bulk-creation -[networkpricing]: https://cloud.google.com/vpc/network-pricing - -> **_WARNING:_** Lenient zone policies can lead to additional egress costs when -> moving large amounts of data between zones in the same region. For example, -> traffic between VMs and traffic from VMs to shared filesystems such as -> Filestore. For more information on egress fees, see the -> [Network Pricing][networkpricing] Google Cloud documentation. -> -> To avoid egress charges, ensure your compute nodes are created in a single -> zone by setting var.zone and leaving var.zones to its default value of the -> empty list. -> -> **_NOTE:_** If a new zone is added to the region while the cluster is active, -> nodes in the partition may be created in that zone. In this case, the -> partition may need to be redeployed to ensure the newly added zone is denied. - -In the zonal example below, the nodeset's zone implicitly defaults to the -deployment variable `vars.zone`: - -```yaml -vars: - zone: us-central1-f - -- id: zonal-nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset -``` - -In the example below, we enable creation in additional zones: - -```yaml -vars: - zone: us-central1-f - -- id: multi-zonal-nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - settings: - zones: - - us-central1-a - - us-central1-b -``` - -## Support -The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform -modules. For support with the underlying modules, see the instructions in the -[slurm-gcp README][slurm-gcp-readme]. - -[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp -[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.4 | -| [google](#requirement\_google) | >= 5.11 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 5.11 | -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | - -## Resources - -| Name | Type | -|------|------| -| [terraform_data.machine_type_zone_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [google_compute_image.slurm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | -| [google_compute_machine_types.machine_types_by_zone](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_machine_types) | data source | -| [google_compute_reservation.reservation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_reservation) | data source | -| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | -| [additional\_disks](#input\_additional\_disks) | Configurations of additional disks to be included on the partition nodes. | 
list(object({
    disk_name                  = optional(string)
    device_name                = optional(string)
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = optional(map(string))
    auto_delete                = optional(bool)
    boot                       = optional(bool)
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = optional(string)
    subnetwork         = string
    subnetwork_project = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    stack_type         = optional(string)
    queue_count        = optional(number)
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | -| [disable\_public\_ips](#input\_disable\_public\_ips) | DEPRECATED: Use `enable_public_ips` instead. | `bool` | `null` | no | -| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | -| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | -| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of boot disk to create for the partition compute nodes. | `number` | `50` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-standard"` | no | -| [dws\_flex](#input\_dws\_flex) | If set and `enabled = true`, will utilize the DWS Flex Start to provision nodes.
See: https://cloud.google.com/blog/products/compute/introducing-dynamic-workload-scheduler
Options:
- enable: Enable DWS Flex Start
- max\_run\_duration: Maximum duration in seconds for the job to run, should not exceed 604,800 (one week).
- use\_job\_duration: Use the job duration to determine the max\_run\_duration, if job duration is not set, max\_run\_duration will be used.
- use\_bulk\_insert: Uses the legacy implementation of DWS Flex Start with Bulk Insert for non-accelerator instances

Limitations:
- CAN NOT be used with reservations;
- CAN NOT be used with placement groups;
- If `use_job_duration` is enabled nodeset can be used in "exclusive" partitions only | 
object({
    enabled          = optional(bool, true)
    max_run_duration = optional(number, 604800) # one week
    use_job_duration = optional(bool, false)
    use_bulk_insert  = optional(bool, false)
  })
| 
{
  "enabled": false
}
| no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_maintenance\_reservation](#input\_enable\_maintenance\_reservation) | Enables slurm reservation for scheduled maintenance. | `bool` | `false` | no | -| [enable\_opportunistic\_maintenance](#input\_enable\_opportunistic\_maintenance) | On receiving maintenance notification, maintenance will be performed as soon as nodes becomes idle. | `bool` | `false` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | -| [enable\_placement](#input\_enable\_placement) | Use placement policy for VMs in this nodeset.
See: https://cloud.google.com/compute/docs/instances/placement-policies-overview
To set max\_distance of used policy, use `placement_max_distance` variable.

Enabled by default, reasons for users to disable it:
- If non-dense reservation is used, user can avoid extra-cost of creating placement policies;
- If user wants to avoid "all or nothing" VM provisioning behaviour;
- If user wants to intentionally have "spread" VMs (e.g. for reliability reasons) | `bool` | `true` | no | -| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true. The node group VMs will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [enable\_spot\_vm](#input\_enable\_spot\_vm) | Enable the partition to use spot VMs (https://cloud.google.com/spot-vms). | `bool` | `false` | no | -| [future\_reservation](#input\_future\_reservation) | If set, will make use of the future reservation for the nodeset. Input can be either the future reservation name or its selfLink in the format 'projects/PROJECT\_ID/zones/ZONE/futureReservations/FUTURE\_RESERVATION\_NAME'.
See https://cloud.google.com/compute/docs/instances/future-reservations-overview | `string` | `""` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm node group VM instances.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-9-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | -| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | -| [instance\_properties](#input\_instance\_properties) | Override the instance properties. Used to test features not supported by Slurm GCP,
recommended for advanced usage only.
See https://cloud.google.com/compute/docs/reference/rest/v1/regionInstances/bulkInsert
If any sub-field (e.g. scheduling) is set, it will override the values computed by
SlurmGCP and ignoring values of provided vars. | `any` | `null` | no | -| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for compute nodes. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to partition compute instances. Key-value pairs. | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Compute Platform machine type to use for this partition compute nodes. | `string` | `"c2-standard-60"` | no | -| [maintenance\_interval](#input\_maintenance\_interval) | Sets the maintenance interval for instances in this nodeset.
See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#maintenance_interval. | `string` | `null` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | -| [name](#input\_name) | Name of the nodeset. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all nodesets. | `string` | n/a | yes | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | -| [node\_conf](#input\_node\_conf) | Map of Slurm node line configuration. | `map(any)` | `{}` | no | -| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of auto-scaling nodes allowed in this partition. | `number` | `10` | no | -| [node\_count\_static](#input\_node\_count\_static) | Number of nodes to be statically created. | `number` | `0` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy.

Note: Placement groups are not supported when on\_host\_maintenance is set to
"MIGRATE" and will be deactivated regardless of the value of
enable\_placement. To support enable\_placement, ensure on\_host\_maintenance is
set to "TERMINATE". | `string` | `"TERMINATE"` | no | -| [placement\_max\_distance](#input\_placement\_max\_distance) | Maximum distance between nodes in the placement group. Requires enable\_placement to be true. Values must be supported by the chosen machine type. | `number` | `null` | no | -| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [region](#input\_region) | The default region for Cloud resources. | `string` | n/a | yes | -| [reservation\_name](#input\_reservation\_name) | Name of the reservation to use for VM resources, should be in one of the following formats:
- projects/PROJECT\_ID/reservations/RESERVATION\_NAME[/reservationBlocks/BLOCK\_ID]
- RESERVATION\_NAME[/reservationBlocks/BLOCK\_ID]

Must be a "SPECIFIC" reservation
Set to empty string if using no reservation or automatically-consumed reservations | `string` | `""` | no | -| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the compute instances. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the compute instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [spot\_instance\_config](#input\_spot\_instance\_config) | Configuration for spot VMs. | 
object({
    termination_action = string
  })
| `null` | no | -| [startup\_script](#input\_startup\_script) | Startup script used by VMs in this nodeset | `string` | `"# no-op"` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | -| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | -| [zone](#input\_zone) | Zone in which to create compute VMs. Additional zones in the same region can be specified in var.zones. | `string` | n/a | yes | -| [zone\_target\_shape](#input\_zone\_target\_shape) | Strategy for distributing VMs across zones in a region.
ANY
GCE picks zones for creating VM instances to fulfill the requested number of VMs
within present resource constraints and to maximize utilization of unused zonal
reservations.
ANY\_SINGLE\_ZONE (default)
GCE always selects a single zone for all the VMs, optimizing for resource quotas,
available reservations and general capacity.
BALANCED
GCE prioritizes acquisition of resources, scheduling VMs in zones where resources
are available while distributing VMs as evenly as possible across allowed zones
to minimize the impact of zonal failure. | `string` | `"ANY_SINGLE_ZONE"` | no | -| [zones](#input\_zones) | Additional zones in which to allow creation of partition nodes. Google Cloud
will find zone based on availability, quota and reservations.
Should not be set if SPECIFIC reservation is used. | `set(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [nodeset](#output\_nodeset) | Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`. | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf deleted file mode 100644 index 5cb0e9751f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/main.tf +++ /dev/null @@ -1,224 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-nodeset", ghpc_role = "compute" }) -} - -module "gpu" { - source = "../../../../modules/internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - metadata = merge( - local.disable_automatic_updates_metadata, - var.metadata - ) - - name = substr(replace(var.name, "/[^a-z0-9]/", ""), 0, 14) - - additional_disks = [ - for ad in var.additional_disks : { - disk_name = ad.disk_name - device_name = ad.device_name - disk_type = ad.disk_type - disk_size_gb = ad.disk_size_gb - disk_labels = merge(ad.disk_labels, local.labels) - auto_delete = ad.auto_delete - boot = ad.boot - disk_resource_manager_tags = ad.disk_resource_manager_tags - } - ] - - public_access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] - access_config = length(var.access_config) == 0 ? local.public_access_config : var.access_config - - service_account = { - email = var.service_account_email - scopes = var.service_account_scopes - } - - ghpc_startup_script = [{ - filename = "ghpc_nodeset_startup.sh" - content = var.startup_script - }] - - termination_action = (var.dws_flex.enabled && !var.dws_flex.use_bulk_insert) ? "DELETE" : try(var.spot_instance_config.termination_action, null) - - nodeset = { - node_count_static = var.node_count_static - node_count_dynamic_max = var.node_count_dynamic_max - node_conf = var.node_conf - nodeset_name = local.name - dws_flex = var.dws_flex - - disk_auto_delete = var.disk_auto_delete - disk_labels = merge(local.labels, var.disk_labels) - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - disk_resource_manager_tags = var.disk_resource_manager_tags - additional_disks = local.additional_disks - - bandwidth_tier = var.bandwidth_tier - can_ip_forward = var.can_ip_forward - - enable_confidential_vm = var.enable_confidential_vm - enable_placement = var.enable_placement - placement_max_distance = var.placement_max_distance - enable_oslogin = var.enable_oslogin - enable_shielded_vm = var.enable_shielded_vm - gpu = one(local.guest_accelerator) - - labels = local.labels - machine_type = terraform_data.machine_type_zone_validation.output - advanced_machine_features = var.advanced_machine_features - metadata = local.metadata - min_cpu_platform = var.min_cpu_platform - - on_host_maintenance = var.on_host_maintenance - preemptible = var.preemptible - region = var.region - resource_manager_tags = var.resource_manager_tags - service_account = local.service_account - shielded_instance_config = var.shielded_instance_config - source_image_family = local.source_image_family # requires source_image_logic.tf - source_image_project = local.source_image_project_normalized # requires source_image_logic.tf - source_image = local.source_image # requires source_image_logic.tf - subnetwork_self_link = var.subnetwork_self_link - additional_networks = var.additional_networks - access_config = local.access_config - tags = var.tags - spot = var.enable_spot_vm - termination_action = local.termination_action - reservation_name = local.reservation_name - future_reservation = local.future_reservation - maintenance_interval = var.maintenance_interval - instance_properties_json = jsonencode(var.instance_properties) - - zone_target_shape = var.zone_target_shape - zone_policy_allow = local.zones - zone_policy_deny = local.zones_deny - - startup_script = local.ghpc_startup_script - network_storage = var.network_storage - - enable_maintenance_reservation = var.enable_maintenance_reservation - enable_opportunistic_maintenance = var.enable_opportunistic_maintenance - } -} - -locals { - zones = setunion(var.zones, [var.zone]) - zones_deny = setsubtract(data.google_compute_zones.available.names, local.zones) -} - -data "google_compute_zones" "available" { - project = var.project_id - region = var.region - - lifecycle { - postcondition { - condition = length(setsubtract(local.zones, self.names)) == 0 - error_message = <<-EOD - Invalid zones=${jsonencode(setsubtract(local.zones, self.names))} - Available zones=${jsonencode(self.names)} - EOD - } - } -} - -locals { - res_match = regex("^(?P(?Pprojects/(?P[a-z0-9-]+)/reservations/)?(?P[a-z0-9-]+)(?P/reservationBlocks/[a-z0-9-]+)?)?$", var.reservation_name) - - res_short_name = local.res_match.name - res_project = coalesce(local.res_match.project, var.project_id) - res_prefix = coalesce(local.res_match.prefix, "projects/${local.res_project}/reservations/") - res_suffix = local.res_match.suffix == null ? "" : local.res_match.suffix - - reservation_name = local.res_match.whole == null ? "" : "${local.res_prefix}${local.res_short_name}${local.res_suffix}" -} - -locals { - fr_match = regex("^(?Pprojects/(?P[a-z0-9-]+)/zones/(?P[a-z0-9-]+)/futureReservations/)?(?P[a-z0-9-]+)?$", var.future_reservation) - - fr_name = local.fr_match.name - fr_project = coalesce(local.fr_match.project, var.project_id) - fr_zone = coalesce(local.fr_match.zone, var.zone) - - future_reservation = var.future_reservation == "" ? "" : "projects/${local.fr_project}/zones/${local.fr_zone}/futureReservations/${local.fr_name}" -} - - -# tflint-ignore: terraform_unused_declarations -data "google_compute_reservation" "reservation" { - count = length(local.reservation_name) > 0 ? 1 : 0 - - name = local.res_short_name - project = local.res_project - zone = var.zone - - lifecycle { - postcondition { - condition = self.self_link != null - error_message = "Couldn't find the reservation ${var.reservation_name}" - } - - postcondition { - condition = coalesce(self.specific_reservation_required, true) - error_message = < 0] -} - -resource "terraform_data" "machine_type_zone_validation" { - input = var.machine_type - lifecycle { - precondition { - condition = length(local.zones_with_machine_type) > 0 - error_message = <<-EOT - machine type ${var.machine_type} is not available in any of the zones ${jsonencode(local.zones)}". To list zones in which it is available, run: - - gcloud compute machine-types list --filter="name=${var.machine_type}" - EOT - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml deleted file mode 100644 index 95b6d1c730..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] -ghpc: - inject_module_id: name - has_to_be_used: true diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf deleted file mode 100644 index d3433bac43..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/outputs.tf +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "nodeset" { - description = "Details of the nodeset. Typically used as input to `schedmd-slurm-gcp-v6-partition`." - value = local.nodeset - - precondition { - condition = !contains([ - "c3-:pd-standard", - "h3-:pd-standard", - "h3-:pd-ssd", - ], "${substr(var.machine_type, 0, 3)}:${var.disk_type}") - error_message = "A disk_type=${var.disk_type} cannot be used with machine_type=${var.machine_type}." - } - - precondition { - condition = var.reservation_name == "" || length(var.zones) == 0 - error_message = <<-EOD - If a reservation is specified, `var.zones` should be empty. - EOD - } - - precondition { - condition = var.placement_max_distance == null || var.enable_placement - error_message = "placement_max_distance requires enable_placement to be set to true." - } - - precondition { - condition = !(startswith(var.machine_type, "a3-") && var.placement_max_distance == 1) - error_message = "A3 machines do not support a placement_max_distance of 1." - } - - precondition { - condition = var.reservation_name == "" || !var.dws_flex.enabled - error_message = "Cannot use reservations with DWS Flex." - } - - precondition { - condition = !var.enable_placement || !var.dws_flex.enabled - error_message = "Cannot use DWS Flex with `enable_placement`." - } - - precondition { - condition = length(var.zones) == 0 || !var.dws_flex.enabled - error_message = <<-EOD - If a DWS Flex is enabled, `var.zones` should be empty. - EOD - } - - precondition { - condition = var.on_host_maintenance == "TERMINATE" || !var.dws_flex.enabled - error_message = "If DWS Flex is used, `on_host_maintenance` should be set to 'TERMINATE'" - } - - precondition { - condition = !var.enable_spot_vm || !var.dws_flex.enabled - error_message = "Cannot use both Flex-Start and Spot VMs for provisioning." - } - - precondition { - condition = var.reservation_name == "" || var.future_reservation == "" - error_message = "Cannot use reservations and future reservations in the same nodeset" - } - - precondition { - condition = !var.enable_placement || var.future_reservation == "" - error_message = "Cannot use `enable_placement` with future reservations." - } - - precondition { - condition = var.future_reservation == "" || length(var.zones) == 0 - error_message = <<-EOD - If a future reservation is specified, `var.zones` should be empty. - EOD - } - - precondition { - condition = var.future_reservation == "" || local.fr_zone == var.zone - error_message = <<-EOD - The zone of the deployment must match that of the future reservation - EOD - } - - precondition { - condition = var.node_count_dynamic_max > 0 || var.node_count_static > 0 - error_message = <<-EOD - This nodeset contains zero nodes, there should be at least one static or dynamic node - EOD - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf deleted file mode 100644 index 0641cea7a8..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/source_image_logic.tf +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # Currently supported images and projects - known_project_families = { - schedmd-slurm-public = [ - "slurm-gcp-6-9-debian-11", - "slurm-gcp-6-9-hpc-rocky-linux-8", - "slurm-gcp-6-9-ubuntu-2004-lts", - "slurm-gcp-6-9-ubuntu-2204-lts-arm64" - ] - } - - # This approach to "hacking" the project name allows a chain of Terraform - # calls to set the instance source_image (boot disk) with a "relative - # resource name" that passes muster with VPC Service Control rules - # - # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 - # https://cloud.google.com/apis/design/resource_names#relative_resource_name - source_image_project_normalized = (can(var.instance_image.family) ? - "projects/${data.google_compute_image.slurm.project}/global/images/family" : - "projects/${data.google_compute_image.slurm.project}/global/images" - ) - source_image_family = can(var.instance_image.family) ? data.google_compute_image.slurm.family : "" - source_image = can(var.instance_image.name) ? data.google_compute_image.slurm.name : "" -} - -data "google_compute_image" "slurm" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = var.instance_image.project - - lifecycle { - precondition { - condition = length(regexall("^projects/.+?/global/images/family$", var.instance_image.project)) == 0 - error_message = "The \"project\" field in var.instance_image no longer supports a long-form ending in \"family\". Specify only the project ID." - } - - postcondition { - condition = var.instance_image_custom || contains(keys(local.known_project_families), self.project) - error_message = <<-EOD - Images in project ${self.project} are not published by SchedMD. Images must be created by compatible releases of the Terraform and Packer modules following the guidance at https://goo.gle/hpc-slurm-images. Set var.instance_image_custom to true to silence this error and acknowledge that you are using a compatible image. - EOD - } - postcondition { - condition = !contains(keys(local.known_project_families), self.project) || try(contains(local.known_project_families[self.project], self.family), false) - error_message = <<-EOD - Image family ${self.family} published by SchedMD in project ${self.project} is not compatible with this release of the Terraform Slurm modules. Select from known compatible releases: - ${join("\n", [for p in try(local.known_project_families[self.project], []) : "\t\"${p}\""])} - EOD - } - postcondition { - condition = var.disk_size_gb >= self.disk_size_gb - error_message = "'disk_size_gb: ${var.disk_size_gb}' is smaller than the image size (${self.disk_size_gb}GB), please increase the blueprint disk size" - } - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf deleted file mode 100644 index 643eca02c4..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/variables.tf +++ /dev/null @@ -1,635 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "name" { - description = <<-EOD - Name of the nodeset. Automatically populated by the module id if not set. - If setting manually, ensure a unique value across all nodesets. - EOD - type = string -} - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "node_conf" { - description = "Map of Slurm node line configuration." - type = map(any) - default = {} - validation { - condition = lookup(var.node_conf, "Sockets", null) == null - error_message = <<-EOD - `Sockets` field is in conflict with `SocketsPerBoard` which is automatically generated by SlurmGCP. - Instead, you can override the following fields: `Boards`, `SocketsPerBoard`, `CoresPerSocket`, and `ThreadsPerCore`. - See: https://slurm.schedmd.com/slurm.conf.html#OPT_Boards and https://slurm.schedmd.com/slurm.conf.html#OPT_Sockets_1 - EOD - } -} - -variable "node_count_static" { - description = "Number of nodes to be statically created." - type = number - default = 0 -} - -variable "node_count_dynamic_max" { - description = "Maximum number of auto-scaling nodes allowed in this partition." - type = number - default = 10 -} - -## VM Definition -variable "instance_template" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: Instance template can not be specified for compute nodes." - type = string - default = null - validation { - condition = var.instance_template == null - error_message = "DEPRECATED: Instance template can not be specified for compute nodes." - } -} - -variable "machine_type" { - description = "Compute Platform machine type to use for this partition compute nodes." - type = string - default = "c2-standard-60" -} - -variable "metadata" { - type = map(string) - description = "Metadata, provided as a map." - default = {} -} - -variable "instance_image" { - description = <<-EOD - Defines the image that will be used in the Slurm node group VM instances. - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - - For more information on creating custom images that comply with Slurm on GCP - see the "Slurm on GCP Custom Images" section in docs/vm-images.md. - EOD - type = map(string) - default = { - family = "slurm-gcp-6-9-hpc-rocky-linux-8" - project = "schedmd-slurm-public" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "instance_image_custom" { - description = <<-EOD - A flag that designates that the user is aware that they are requesting - to use a custom and potentially incompatible image for this Slurm on - GCP module. - - If the field is set to false, only the compatible families and project - names will be accepted. The deployment will fail with any other image - family or name. If set to true, no checks will be done. - - See: https://goo.gle/hpc-slurm-images - EOD - type = bool - default = false -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "tags" { - type = list(string) - description = "Network tag list." - default = [] -} - -variable "disk_type" { - description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." - type = string - default = "pd-standard" -} - -variable "disk_size_gb" { - description = "Size of boot disk to create for the partition compute nodes." - type = number - default = 50 -} - -variable "disk_auto_delete" { - type = bool - description = "Whether or not the boot disk should be auto-deleted." - default = true -} - -variable "disk_labels" { - description = "Labels specific to the boot disk. These will be merged with var.labels." - type = map(string) - default = {} -} - -variable "disk_resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} - validation { - condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) - error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" - } - validation { - condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) - error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" - } -} - -variable "additional_disks" { - description = "Configurations of additional disks to be included on the partition nodes." - type = list(object({ - disk_name = optional(string) - device_name = optional(string) - disk_size_gb = optional(number) - disk_type = optional(string) - disk_labels = optional(map(string)) - auto_delete = optional(bool) - boot = optional(bool) - disk_resource_manager_tags = optional(map(string)) - })) - default = [] -} - -variable "enable_confidential_vm" { - type = bool - description = "Enable the Confidential VM configuration. Note: the instance image must support option." - default = false -} - -variable "enable_shielded_vm" { - type = bool - description = "Enable the Shielded VM configuration. Note: the instance image must support option." - default = false -} - -variable "shielded_instance_config" { - type = object({ - enable_integrity_monitoring = bool - enable_secure_boot = bool - enable_vtpm = bool - }) - description = <<-EOD - Shielded VM configuration for the instance. Note: not used unless - enable_shielded_vm is 'true'. - - enable_integrity_monitoring : Compare the most recent boot measurements to the - integrity policy baseline and return a pair of pass/fail results depending on - whether they match or not. - - enable_secure_boot : Verify the digital signature of all boot components, and - halt the boot process if signature verification fails. - - enable_vtpm : Use a virtualized trusted platform module, which is a - specialized computer chip you can use to encrypt objects like keys and - certificates. - EOD - default = { - enable_integrity_monitoring = true - enable_secure_boot = true - enable_vtpm = true - } -} - - -variable "enable_oslogin" { - type = bool - description = <<-EOD - Enables Google Cloud os-login for user login and authentication for VMs. - See https://cloud.google.com/compute/docs/oslogin - EOD - default = true -} - -variable "can_ip_forward" { - description = "Enable IP forwarding, for NAT instances for example." - type = bool - default = false -} - -variable "advanced_machine_features" { - description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" - type = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) - default = { - threads_per_core = 1 # disable SMT by default - } -} - -variable "resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} - validation { - condition = alltrue([for value in var.resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) - error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" - } - validation { - condition = alltrue([for value in keys(var.resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) - error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" - } -} - -variable "enable_smt" { # tflint-ignore: terraform_unused_declarations - type = bool - description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - default = null - validation { - condition = var.enable_smt == null - error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - } -} - -variable "labels" { - description = "Labels to add to partition compute instances. Key-value pairs." - type = map(string) - default = {} -} - -variable "min_cpu_platform" { - description = "The name of the minimum CPU platform that you want the instance to use." - type = string - default = null -} - -variable "on_host_maintenance" { - type = string - description = <<-EOD - Instance availability Policy. - - Note: Placement groups are not supported when on_host_maintenance is set to - "MIGRATE" and will be deactivated regardless of the value of - enable_placement. To support enable_placement, ensure on_host_maintenance is - set to "TERMINATE". - EOD - default = "TERMINATE" -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = string, - count = number - })) - default = [] - nullable = false - - validation { - condition = length(var.guest_accelerator) <= 1 - error_message = "The Slurm modules supports 0 or 1 models of accelerator card on each node." - } -} - -variable "preemptible" { - description = "Should use preemptibles to burst." - type = bool - default = false -} - - -variable "service_account_email" { - description = "Service account e-mail address to attach to the compute instances." - type = string - default = null -} - -variable "service_account_scopes" { - description = "Scopes to attach to the compute instances." - type = set(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -variable "service_account" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." - type = object({ - email = string - scopes = set(string) - }) - default = null - validation { - condition = var.service_account == null - error_message = "DEPRECATED: Use `service_account_email` and `service_account_scopes` instead." - } -} - -variable "enable_spot_vm" { - description = "Enable the partition to use spot VMs (https://cloud.google.com/spot-vms)." - type = bool - default = false -} - -variable "spot_instance_config" { - description = "Configuration for spot VMs." - type = object({ - termination_action = string - }) - default = null -} - -variable "bandwidth_tier" { - description = < 0 - error_message = "Reservation name must be either empty or in the format '[projects/PROJECT_ID/reservations/]RESERVATION_NAME[/reservationBlocks/BLOCK_ID]', [...] are optional parts." - } -} - -variable "future_reservation" { - description = <<-EOD - If set, will make use of the future reservation for the nodeset. Input can be either the future reservation name or its selfLink in the format 'projects/PROJECT_ID/zones/ZONE/futureReservations/FUTURE_RESERVATION_NAME'. - See https://cloud.google.com/compute/docs/instances/future-reservations-overview - EOD - type = string - default = "" - nullable = false - - validation { - condition = length(regexall("^(projects/([a-z0-9-]+)/zones/([a-z0-9-]+)/futureReservations/([a-z0-9-]+))?$", var.future_reservation)) > 0 || length(regexall("^([a-z0-9-]+)$", var.future_reservation)) > 0 - error_message = "Future reservation must be either the future reservation name or its selfLink in the format 'projects/PROJECT_ID/zone/ZONE/futureReservations/FUTURE_RESERVATION_NAME'." - } -} - -variable "maintenance_interval" { - description = <<-EOD - Sets the maintenance interval for instances in this nodeset. - See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#maintenance_interval. - EOD - type = string - default = null -} - -variable "startup_script" { - description = "Startup script used by VMs in this nodeset" - type = string - default = "# no-op" -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured on nodes." - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - })) - default = [] -} - - -variable "instance_properties" { - description = <<-EOD - Override the instance properties. Used to test features not supported by Slurm GCP, - recommended for advanced usage only. - See https://cloud.google.com/compute/docs/reference/rest/v1/regionInstances/bulkInsert - If any sub-field (e.g. scheduling) is set, it will override the values computed by - SlurmGCP and ignoring values of provided vars. - EOD - type = any - default = null -} - - -variable "enable_maintenance_reservation" { - type = bool - description = "Enables slurm reservation for scheduled maintenance." - default = false -} - - -variable "enable_opportunistic_maintenance" { - type = bool - description = "On receiving maintenance notification, maintenance will be performed as soon as nodes becomes idle." - default = false -} - - -variable "dws_flex" { - description = <<-EOD - If set and `enabled = true`, will utilize the DWS Flex Start to provision nodes. - See: https://cloud.google.com/blog/products/compute/introducing-dynamic-workload-scheduler - Options: - - enable: Enable DWS Flex Start - - max_run_duration: Maximum duration in seconds for the job to run, should not exceed 604,800 (one week). - - use_job_duration: Use the job duration to determine the max_run_duration, if job duration is not set, max_run_duration will be used. - - use_bulk_insert: Uses the legacy implementation of DWS Flex Start with Bulk Insert for non-accelerator instances - - Limitations: - - CAN NOT be used with reservations; - - CAN NOT be used with placement groups; - - If `use_job_duration` is enabled nodeset can be used in "exclusive" partitions only - - EOD - - type = object({ - enabled = optional(bool, true) - max_run_duration = optional(number, 604800) # one week - use_job_duration = optional(bool, false) - use_bulk_insert = optional(bool, false) - }) - default = { - enabled = false - } - validation { - condition = var.dws_flex.max_run_duration >= 600 && var.dws_flex.max_run_duration <= 604800 - error_message = "Max duration must be at least than 10 minutes, and cannot be more than one week." - } -} - -variable "placement_max_distance" { - type = number - description = "Maximum distance between nodes in the placement group. Requires enable_placement to be true. Values must be supported by the chosen machine type." - nullable = true - default = null - - validation { - condition = coalesce(var.placement_max_distance, 1) >= 1 && coalesce(var.placement_max_distance, 3) <= 3 - error_message = "Invalid value for placement_max_distance. Valid values are null, 1, 2, or 3." - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf deleted file mode 100644 index b879e1bd17..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.4" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 5.11" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-nodeset/v1.57.0" - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md deleted file mode 100644 index 61167a9f1e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md +++ /dev/null @@ -1,105 +0,0 @@ -## Description - -This module creates a compute partition that can be used as input to the -[schedmd-slurm-gcp-v6-controller](../../scheduler/schedmd-slurm-gcp-v6-controller/README.md). - -The partition module is designed to work alongside the -[schedmd-slurm-gcp-v6-nodeset](../schedmd-slurm-gcp-v6-nodeset/README.md) -module. A partition can be made up of one or -more nodesets, provided either through `use` (preferred) or defined manually -in the `nodeset` variable. - -### Example - -The following code snippet creates a partition module with: - -* 2 nodesets added via `use`. - * The first nodeset is made up of machines of type `c2-standard-30`. - * The second nodeset is made up of machines of type `c2-standard-60`. - * Both nodesets have a maximum count of 200 dynamically created nodes. -* partition name of "compute". -* connected to the `network` module via `use`. -* nodes mounted to homefs via `use`. - -```yaml -- id: nodeset_1 - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: - - network - settings: - name: c30 - node_count_dynamic_max: 200 - machine_type: c2-standard-30 - -- id: nodeset_2 - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: - - network - settings: - name: c60 - node_count_dynamic_max: 200 - machine_type: c2-standard-60 - -- id: compute_partition - source: community/modules/compute/schedmd-slurm-gcp-v6-partition - use: - - homefs - - nodeset_1 - - nodeset_2 - settings: - partition_name: compute -``` - -## Support - -The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform -modules. For support with the underlying modules, see the instructions in the -[slurm-gcp README][slurm-gcp-readme]. - -[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp -[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [exclusive](#input\_exclusive) | Exclusive job access to nodes. When set to true nodes execute single job and are deleted
after job exits. If set to false, multiple jobs can be scheduled on one node. | `bool` | `true` | no | -| [is\_default](#input\_is\_default) | Sets this partition as the default partition by updating the partition\_conf.
If "Default" is already set in partition\_conf, this variable will have no effect. | `bool` | `false` | no | -| [network\_storage](#input\_network\_storage) | DEPRECATED | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [nodeset](#input\_nodeset) | A list of nodesets.
For type definition see community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf::nodeset | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 1)
    node_conf              = optional(map(string), {})
    nodeset_name           = string
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    bandwidth_tier                   = optional(string, "platform_default")
    can_ip_forward                   = optional(bool, false)
    disk_auto_delete                 = optional(bool, true)
    disk_labels                      = optional(map(string), {})
    disk_resource_manager_tags       = optional(map(string), {})
    disk_size_gb                     = optional(number)
    disk_type                        = optional(string)
    enable_confidential_vm           = optional(bool, false)
    enable_placement                 = optional(bool, false)
    placement_max_distance           = optional(number, null)
    enable_oslogin                   = optional(bool, true)
    enable_shielded_vm               = optional(bool, false)
    enable_maintenance_reservation   = optional(bool, false)
    enable_opportunistic_maintenance = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    dws_flex = object({
      enabled          = bool
      max_run_duration = number
      use_job_duration = bool
      use_bulk_insert  = bool
    })
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    maintenance_interval     = optional(string)
    instance_properties_json = string
    metadata                 = optional(map(string), {})
    min_cpu_platform         = optional(string)
    network_tier             = optional(string, "STANDARD")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    subnetwork_self_link = string
    additional_networks = optional(list(object({
      network            = string
      subnetwork         = string
      subnetwork_project = string
      network_ip         = string
      nic_type           = string
      stack_type         = string
      queue_count        = number
      access_config = list(object({
        nat_ip       = string
        network_tier = string
      }))
      ipv6_access_config = list(object({
        network_tier = string
      }))
      alias_ip_range = list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      }))
    })))
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    spot               = optional(bool, false)
    tags               = optional(list(string), [])
    termination_action = optional(string)
    reservation_name   = optional(string)
    future_reservation = string
    startup_script = optional(list(object({
      filename = string
    content = string })), [])

    zone_target_shape = string
    zone_policy_allow = set(string)
    zone_policy_deny  = set(string)
  }))
| `[]` | no | -| [nodeset\_dyn](#input\_nodeset\_dyn) | Defines dynamic nodesets, as a list. | 
list(object({
    nodeset_name    = string
    nodeset_feature = string
  }))
| `[]` | no | -| [nodeset\_tpu](#input\_nodeset\_tpu) | Define TPU nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 5)
    nodeset_name           = string
    enable_public_ip       = optional(bool, false)
    node_type              = string
    accelerator_config = optional(object({
      topology = string
      version  = string
      }), {
      topology = ""
      version  = ""
    })
    tf_version   = string
    preemptible  = optional(bool, false)
    preserve_tpu = optional(bool, false)
    zone         = string
    data_disks   = optional(list(string), [])
    docker_image = optional(string, "")
    network_storage = optional(list(object({
      server_ip     = string
      remote_mount  = string
      local_mount   = string
      fs_type       = string
      mount_options = string
    })), [])
    subnetwork = string
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    project_id = string
    reserved   = optional(string, false)
  }))
| `[]` | no | -| [partition\_conf](#input\_partition\_conf) | Slurm partition configuration as a map.
See https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION | `map(string)` | `{}` | no | -| [partition\_name](#input\_partition\_name) | The name of the slurm partition. | `string` | n/a | yes | -| [resume\_timeout](#input\_resume\_timeout) | Maximum time permitted (in seconds) between when a node resume request is issued and when the node is actually available for use.
If null is given, then a smart default will be chosen depending on nodesets in partition.
This sets 'ResumeTimeout' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_ResumeTimeout_1 for details. | `number` | `null` | no | -| [suspend\_time](#input\_suspend\_time) | Nodes which remain idle or down for this number of seconds will be placed into power save mode by SuspendProgram.
This sets 'SuspendTime' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTime_1 for details.
NOTE: use value -1 to exclude partition from suspend.
NOTE 2: if `var.exclusive` is set to true (default), nodes are deleted immediately after job finishes. | `number` | `300` | no | -| [suspend\_timeout](#input\_suspend\_timeout) | Maximum time permitted (in seconds) between when a node suspend request is issued and when the node is shutdown.
If null is given, then a smart default will be chosen depending on nodesets in partition.
This sets 'SuspendTimeout' in partition\_conf.
See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTimeout_1 for details. | `number` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [nodeset](#output\_nodeset) | Details of a nodesets in this partition | -| [nodeset\_dyn](#output\_nodeset\_dyn) | Details of a dynamic nodesets in this partition | -| [nodeset\_tpu](#output\_nodeset\_tpu) | Details of a TPU nodesets in this partition | -| [partitions](#output\_partitions) | Details of a slurm partition | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf deleted file mode 100644 index 35b4ca3c19..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/main.tf +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - use_static = [for ns in concat(var.nodeset, var.nodeset_tpu) : ns.nodeset_name if ns.node_count_static > 0] - uses_job_duration = length([for ns in var.nodeset : ns.dws_flex.use_job_duration if ns.dws_flex.use_job_duration]) > 0 - - has_node = length(var.nodeset) > 0 - has_dyn = length(var.nodeset_dyn) > 0 - has_tpu = length(var.nodeset_tpu) > 0 - has_flex = length([for ns in var.nodeset : ns.dws_flex.enabled if ns.dws_flex.enabled]) > 0 -} - -locals { - partition_conf = merge({ - "Default" = var.is_default ? "YES" : null - "SuspendTime" = var.suspend_time < 0 ? "INFINITE" : var.suspend_time - "SuspendTimeout" = var.suspend_timeout != null ? var.suspend_timeout : (local.has_tpu ? 240 : 120) - }, var.partition_conf, { "ResumeTimeout" = local.has_flex ? 65535 : try(var.partition_conf["ResumeTimeout"], coalesce(var.resume_timeout, (local.has_tpu ? 600 : 300))) }) - - partition = { - partition_name = var.partition_name - partition_conf = local.partition_conf - - partition_nodeset = [for ns in var.nodeset : ns.nodeset_name] - partition_nodeset_tpu = [for ns in var.nodeset_tpu : ns.nodeset_name] - partition_nodeset_dyn = [for ns in var.nodeset_dyn : ns.nodeset_name] - # Options - enable_job_exclusive = var.exclusive - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml deleted file mode 100644 index 13ea127b3c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] -ghpc: - has_to_be_used: true diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf deleted file mode 100644 index ff796ad0f6..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/outputs.tf +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "partitions" { - description = "Details of a slurm partition" - - value = [local.partition] - - precondition { - condition = (length(local.use_static) == 0) || !var.exclusive - error_message = <<-EOD - Can't use static nodes within partition with `var.exclusive` set to `true`. - NOTE: Partition's `var.exclusive` is set to `true` by default. Set it to `false` explicitly to use static nodes. - EOD - } - - precondition { - # Can not mix TPU with other non-TPU nodesets due to SlurmGCP specific limitations; - # Can not mix dynamic with non-dynamic nodesets due to Slurms inability to - # turn off "power management" at nodeset level (can only do it at partition or node level). - condition = sum([for b in [local.has_node, local.has_dyn, local.has_tpu] : b ? 1 : 0]) == 1 - error_message = "Partition must contain exactly one type of nodeset." - } - - precondition { - condition = !local.uses_job_duration || var.exclusive - error_message = "`use_job_duration` can only be used in exclusive partitions" - } -} - -output "nodeset" { - description = "Details of a nodesets in this partition" - - value = var.nodeset -} - -output "nodeset_tpu" { - description = "Details of a TPU nodesets in this partition" - - value = var.nodeset_tpu -} - - -output "nodeset_dyn" { - description = "Details of a dynamic nodesets in this partition" - - value = var.nodeset_dyn -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf deleted file mode 100644 index 338d90954f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/variables.tf +++ /dev/null @@ -1,310 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "partition_name" { - description = "The name of the slurm partition." - type = string - - validation { - condition = can(regex("^[a-z](?:[a-z0-9]*)$", var.partition_name)) - error_message = "Variable 'partition_name' must be a match of regex '^[a-z](?:[a-z0-9]*)$'." - } -} - -variable "partition_conf" { - description = <<-EOD - Slurm partition configuration as a map. - See https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION - EOD - type = map(string) - default = {} -} - -variable "is_default" { - description = <<-EOD - Sets this partition as the default partition by updating the partition_conf. - If "Default" is already set in partition_conf, this variable will have no effect. - EOD - type = bool - default = false -} - -variable "exclusive" { - description = <<-EOD - Exclusive job access to nodes. When set to true nodes execute single job and are deleted - after job exits. If set to false, multiple jobs can be scheduled on one node. - EOD - type = bool - default = true -} - -variable "nodeset" { - description = <<-EOD - A list of nodesets. - For type definition see community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf::nodeset - EOD - type = list(object({ - node_count_static = optional(number, 0) - node_count_dynamic_max = optional(number, 1) - node_conf = optional(map(string), {}) - nodeset_name = string - additional_disks = optional(list(object({ - disk_name = optional(string) - device_name = optional(string) - disk_size_gb = optional(number) - disk_type = optional(string) - disk_labels = optional(map(string), {}) - auto_delete = optional(bool, true) - boot = optional(bool, false) - disk_resource_manager_tags = optional(map(string), {}) - })), []) - bandwidth_tier = optional(string, "platform_default") - can_ip_forward = optional(bool, false) - disk_auto_delete = optional(bool, true) - disk_labels = optional(map(string), {}) - disk_resource_manager_tags = optional(map(string), {}) - disk_size_gb = optional(number) - disk_type = optional(string) - enable_confidential_vm = optional(bool, false) - enable_placement = optional(bool, false) - placement_max_distance = optional(number, null) - enable_oslogin = optional(bool, true) - enable_shielded_vm = optional(bool, false) - enable_maintenance_reservation = optional(bool, false) - enable_opportunistic_maintenance = optional(bool, false) - gpu = optional(object({ - count = number - type = string - })) - dws_flex = object({ - enabled = bool - max_run_duration = number - use_job_duration = bool - use_bulk_insert = bool - }) - labels = optional(map(string), {}) - machine_type = optional(string) - advanced_machine_features = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) - maintenance_interval = optional(string) - instance_properties_json = string - metadata = optional(map(string), {}) - min_cpu_platform = optional(string) - network_tier = optional(string, "STANDARD") - network_storage = optional(list(object({ - server_ip = string - remote_mount = string - local_mount = string - fs_type = string - mount_options = string - client_install_runner = optional(map(string)) - mount_runner = optional(map(string)) - })), []) - on_host_maintenance = optional(string) - preemptible = optional(bool, false) - region = optional(string) - resource_manager_tags = optional(map(string), {}) - service_account = optional(object({ - email = optional(string) - scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) - })) - shielded_instance_config = optional(object({ - enable_integrity_monitoring = optional(bool, true) - enable_secure_boot = optional(bool, true) - enable_vtpm = optional(bool, true) - })) - source_image_family = optional(string) - source_image_project = optional(string) - source_image = optional(string) - subnetwork_self_link = string - additional_networks = optional(list(object({ - network = string - subnetwork = string - subnetwork_project = string - network_ip = string - nic_type = string - stack_type = string - queue_count = number - access_config = list(object({ - nat_ip = string - network_tier = string - })) - ipv6_access_config = list(object({ - network_tier = string - })) - alias_ip_range = list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })) - }))) - access_config = optional(list(object({ - nat_ip = string - network_tier = string - }))) - spot = optional(bool, false) - tags = optional(list(string), []) - termination_action = optional(string) - reservation_name = optional(string) - future_reservation = string - startup_script = optional(list(object({ - filename = string - content = string })), []) - - zone_target_shape = string - zone_policy_allow = set(string) - zone_policy_deny = set(string) - })) - default = [] - - validation { - condition = length(distinct(var.nodeset[*].nodeset_name)) == length(var.nodeset) - error_message = "All nodesets must have a unique name." - } -} - -variable "nodeset_tpu" { - description = "Define TPU nodesets, as a list." - type = list(object({ - node_count_static = optional(number, 0) - node_count_dynamic_max = optional(number, 5) - nodeset_name = string - enable_public_ip = optional(bool, false) - node_type = string - accelerator_config = optional(object({ - topology = string - version = string - }), { - topology = "" - version = "" - }) - tf_version = string - preemptible = optional(bool, false) - preserve_tpu = optional(bool, false) - zone = string - data_disks = optional(list(string), []) - docker_image = optional(string, "") - network_storage = optional(list(object({ - server_ip = string - remote_mount = string - local_mount = string - fs_type = string - mount_options = string - })), []) - subnetwork = string - service_account = optional(object({ - email = optional(string) - scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) - })) - project_id = string - reserved = optional(string, false) - })) - default = [] - - validation { - condition = length(distinct([for x in var.nodeset_tpu : x.nodeset_name])) == length(var.nodeset_tpu) - error_message = "All TPU nodesets must have a unique name." - } -} - -variable "nodeset_dyn" { - description = "Defines dynamic nodesets, as a list." - type = list(object({ - nodeset_name = string - nodeset_feature = string - })) - default = [] - - validation { - condition = length(distinct([for x in var.nodeset_dyn : x.nodeset_name])) == length(var.nodeset_dyn) - error_message = "All dynamic nodesets must have a unique name." - } -} - -variable "resume_timeout" { - description = <<-EOD - Maximum time permitted (in seconds) between when a node resume request is issued and when the node is actually available for use. - If null is given, then a smart default will be chosen depending on nodesets in partition. - This sets 'ResumeTimeout' in partition_conf. - See https://slurm.schedmd.com/slurm.conf.html#OPT_ResumeTimeout_1 for details. - EOD - type = number - default = null - - validation { - condition = var.resume_timeout == null ? true : var.resume_timeout > 0 && var.resume_timeout < 65536 - error_message = "Value must be > 0 and < 65536" - } -} - -variable "suspend_time" { - description = <<-EOD - Nodes which remain idle or down for this number of seconds will be placed into power save mode by SuspendProgram. - This sets 'SuspendTime' in partition_conf. - See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTime_1 for details. - NOTE: use value -1 to exclude partition from suspend. - NOTE 2: if `var.exclusive` is set to true (default), nodes are deleted immediately after job finishes. - EOD - type = number - default = 300 - - validation { - condition = var.suspend_time >= -1 - error_message = "Value must be >= -1." - } -} - -variable "suspend_timeout" { - description = <<-EOD - Maximum time permitted (in seconds) between when a node suspend request is issued and when the node is shutdown. - If null is given, then a smart default will be chosen depending on nodesets in partition. - This sets 'SuspendTimeout' in partition_conf. - See https://slurm.schedmd.com/slurm.conf.html#OPT_SuspendTimeout_1 for details. - EOD - type = number - default = null - - validation { - condition = var.suspend_timeout == null ? true : var.suspend_timeout > 0 - error_message = "Value must be > 0." - } -} - - -# tflint-ignore: terraform_unused_declarations -variable "network_storage" { - description = "DEPRECATED" - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] - validation { - condition = length(var.network_storage) == 0 - error_message = <<-EOD - network_storage in partition module is deprecated and should not be set. - To add network storage to compute nodes, use network_storage of nodeset module instead. - EOD - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf deleted file mode 100644 index 5298c89dbe..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/compute/schedmd-slurm-gcp-v6-partition/versions.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.3" - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v6-partition/v1.57.0" - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/README.md deleted file mode 100644 index 994f1500ba..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/README.md +++ /dev/null @@ -1,157 +0,0 @@ -## Description - -This module provides ways to create and manage Google Cloud Artifact Registry repositories. - -Currently this module is built to support repositories in Docker format although there are placeholder variables for other types which may work too. Remote repositories with pull-through cache functionality integrated with Google Secret Manager is currently supported. The aim of this module is to eventually offer feature parity with this [Terraform module](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/artifact_registry_repository#nested_remote_repository_config), allowing creation of repositories in various formats, including Docker, Maven, NPM, Python, APT, YUM, and COMMON. - -This module is best suited for managing artifact repositories in HPC/AI containerized environments where artifacts need to be shared across distributed systems. It includes IAM role configurations and secret access handling for seamless integration with CI/CD pipelines and other services too. - -It is designed to help facilitate containerized workloads running in the Cluster Toolkit with SLURM leveraging [Enroot](https://github.com/NVIDIA/enroot) and [Pyxis](https://github.com/NVIDIA/pyxis). Docker repositories can store container images that are used in job submissions, enabling efficient and scalable execution of containerized HPC or AI based workloads. - -## Usage - -### Service Account / APIs - -You will need to enable the relevant APIs and create a Service Account for your cluster with the following Artifact Registry permissions. - -```yaml - - id: services-api - source: community/modules/project/service-enablement - settings: - gcp_service_list: - - secretmanager.googleapis.com - - cloudbuild.googleapis.com - - artifactregistry.googleapis.com - - - source: community/modules/project/service-account - kind: terraform - id: hpc_service_account - settings: - project_id: project_name - name: service_account_name - project_roles: - - artifactregistry.reader - - artifactregistry.writer - - secretmanager.secretAccessor -``` - -### Deployment - -Create a standard Docker repository. - -```yaml -- id: registry - source: community/modules/container/artifact-registry - settings: - repo_mode: STANDARD_REPOSITORY - format: DOCKER -``` - -Mirror of public Docker Hub repository. - -```yaml -- id: dockerhub_registry - source: community/modules/container/artifact-registry - settings: - repo_mode: REMOTE_REPOSITORY - format: DOCKER - repo_public_repository: DOCKER_HUB -``` - -Mirror of NVIDIA's [NGC Catalog](https://catalog.ngc.nvidia.com/containers). [API key](https://org.ngc.nvidia.com/setup/api-key) used in blueprint is stored in Secret Manager. - -```yaml -- id: ngc_registry - source: community/modules/container/artifact-registry - settings: - repo_mode: REMOTE_REPOSITORY - format: DOCKER - repo_mirror_url: "https://nvcr.io" - repo_username: $oauthtoken - repo_password: api_key_here - use_upstream_credentials: True -``` - -### Container Operations - -Retrieve `$REPOSITORY_NAME` from [Artifact Registry](https://console.cloud.google.com/artifacts) or by using `gcloud`. - -```yaml -gcloud artifacts repositories list --project="${PROJECT_ID}" -``` - -Pulling containers from your mirrored internal Artifact Repositories. - -Pull [Ubuntu](https://hub.docker.com/_/ubuntu) from Docker Hub mirror. - -```yaml -docker pull ${REGION}-docker.pkg.dev/${PROJECT_NAME}/${REPOSITORY_NAME}/library/ubuntu:latest -``` - -Pull [Pytorch](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch) from NGC Catalog mirror. - -```yaml -docker pull ${REGION}-docker.pkg.dev/${PROJECT_NAME}/${REPOSITORY_NAME}/nvidia/pytorch:24.11-py3 -``` - -Alternatively, proceed with running SLURM's [NVIDIA/pyxis](https://github.com/NVIDIA/pyxis) plugin, which will now be able to pull and use these containers directly from the mirrored repositories. - -Note: only Docker registries have been tested so far. Placeholders do exist for other registry types which may or may not work. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [random](#provider\_random) | ~> 3.0 | -| [terraform](#provider\_terraform) | n/a | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_artifact_registry_repository.artifact_registry](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/artifact_registry_repository) | resource | -| [google_secret_manager_secret.repo_password_secret](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | -| [google_secret_manager_secret_version.repo_password_secret_version](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [random_password.repo_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | -| [terraform_data.input_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment. | `string` | n/a | yes | -| [format](#input\_format) | Artifact Registry format (e.g., DOCKER). | `string` | `"DOCKER"` | no | -| [labels](#input\_labels) | Labels to add to the artifact registry. Key-value pairs. | `map(string)` | `{}` | no | -| [project\_id](#input\_project\_id) | Project ID where the artifact registry and secret are created. | `string` | n/a | yes | -| [region](#input\_region) | Region for the artifact registry. | `string` | n/a | yes | -| [repo\_mirror\_url](#input\_repo\_mirror\_url) | For REMOTE\_REPOSITORY, URL for a custom or common mirror. | `string` | `null` | no | -| [repo\_mode](#input\_repo\_mode) | Artifact Registry mode (STANDARD\_REPOSITORY, REMOTE\_REPOSITORY, etc.). | `string` | `"STANDARD_REPOSITORY"` | no | -| [repo\_password](#input\_repo\_password) | Optional password/API key. If null, one will be randomly generated. | `string` | `null` | no | -| [repo\_public\_repository](#input\_repo\_public\_repository) | For REMOTE\_REPOSITORY, name of a known public repo as per the Terraform module
(e.g., DOCKER\_HUB) or null for custom repo. | `string` | `null` | no | -| [repo\_username](#input\_repo\_username) | Username for external repository. | `string` | `null` | no | -| [repository\_base](#input\_repository\_base) | For APT/YUM public repos, repository\_base (e.g., 'DEBIAN', 'UBUNTU'). | `string` | `null` | no | -| [repository\_path](#input\_repository\_path) | For APT/YUM public repos, repository\_path (e.g., 'debian/dists/buster'). | `string` | `null` | no | -| [use\_upstream\_credentials](#input\_use\_upstream\_credentials) | Configure Service Account to use upstream credentials for REMOTE\_REPOSITORY:
If true, a username/password is used for the REMOTE\_REPOSITORY mirror.
If false (or if repo\_password == null), no password is created at all.
Note: Blueprint credentials will be stored in Secrets Manager. | `bool` | `false` | no | -| [user\_managed\_replication](#input\_user\_managed\_replication) | (Optional) A list of objects to enable user-managed replication.
Each object can have:
location = string
kms\_key\_name = optional(string)
If empty, auto replication is used. | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [registry\_url](#output\_registry\_url) | The URL of the created artifact registry. | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/main.tf deleted file mode 100644 index c3406af607..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/main.tf +++ /dev/null @@ -1,268 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "artifact-registry", ghpc_role = "container" }) -} - -locals { - # Auto (i.e., empty) vs user-managed replication - auto = length(var.user_managed_replication) == 0 ? true : false - - # For remote custom repositories, parse out host to create a base_component name - mirror_url_no_proto = var.repo_mirror_url != null ? replace(replace(var.repo_mirror_url, "https://", ""), "http://", "") : "" - mirror_host = local.mirror_url_no_proto != "" ? split("/", local.mirror_url_no_proto)[0] : "" - - base_component = replace( - replace( - replace( - lower( - local.mirror_host != "" - ? "${var.format}-${var.repo_mode}-${local.mirror_host}" - : "${var.format}-${var.repo_mode}-nohost" - ), - "\\.", "-" - ), - "/", "-" - ), - "_", "-" - ) - - repository_suffix = random_id.resource_name_suffix.hex - - # The final name for the artifact registry repository - repository_name = replace( - replace( - lower( - format("%s-%s", local.base_component, local.repository_suffix) - ), - ".", "-" - ), - "/", "-" - ) - - # The secret name is derived from the repository name - # with a suffix like "-secret". - derived_secret_name = format("%s-secret", local.repository_name) -} - -############################## -# PASSWORD / SECRET -############################## - -# Only create a random password if user didn't supply one -resource "random_password" "repo_password" { - count = var.use_upstream_credentials && var.repo_password == null ? 1 : 0 - length = 24 - special = true - override_special = "_-#=." -} - -resource "google_secret_manager_secret" "repo_password_secret" { - count = var.use_upstream_credentials ? 1 : 0 - project = var.project_id - - # Derive the secret ID from the repository name - secret_id = local.derived_secret_name - - labels = local.labels - - replication { - dynamic "auto" { - for_each = local.auto ? [1] : [] - content {} - } - dynamic "user_managed" { - for_each = local.auto ? [] : [1] - content { - dynamic "replicas" { - for_each = var.user_managed_replication - content { - location = replicas.value.location - dynamic "customer_managed_encryption" { - for_each = replicas.value.kms_key_name != null ? [1] : [] - content { - kms_key_name = customer_managed_encryption.value - } - } - } - } - } - } - } -} - -resource "google_secret_manager_secret_version" "repo_password_secret_version" { - count = var.use_upstream_credentials ? 1 : 0 - secret = google_secret_manager_secret.repo_password_secret[0].id - - # If user provided a password, use it. Otherwise use the random password. - secret_data = var.repo_password != null ? var.repo_password : random_password.repo_password[0].result -} - -############################## -# IAM BINDINGS -############################## - -############################## -# ARTIFACT REGISTRY -############################## - -resource "random_id" "resource_name_suffix" { - byte_length = 2 -} - -resource "google_artifact_registry_repository" "artifact_registry" { - project = var.project_id - location = var.region - format = var.format - mode = var.repo_mode - description = var.deployment_name - labels = local.labels - repository_id = local.repository_name - - # Only create remote_repository_config if REMOTE_REPOSITORY - dynamic "remote_repository_config" { - for_each = var.repo_mode == "REMOTE_REPOSITORY" ? [1] : [] - content { - description = "Pull-through cache" - - dynamic "docker_repository" { - for_each = var.format == "DOCKER" && var.repo_public_repository != null ? [1] : [] - content { - public_repository = var.repo_public_repository - } - } - - dynamic "docker_repository" { - for_each = var.format == "DOCKER" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "maven_repository" { - for_each = var.format == "MAVEN" && var.repo_public_repository != null ? [1] : [] - content { - public_repository = var.repo_public_repository - } - } - - dynamic "maven_repository" { - for_each = var.format == "MAVEN" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "npm_repository" { - for_each = var.format == "NPM" && var.repo_public_repository != null ? [1] : [] - content { - public_repository = var.repo_public_repository - } - } - - dynamic "npm_repository" { - for_each = var.format == "NPM" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "python_repository" { - for_each = var.format == "PYTHON" && var.repo_public_repository != null ? [1] : [] - content { - public_repository = var.repo_public_repository - } - } - - dynamic "python_repository" { - for_each = var.format == "PYTHON" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "apt_repository" { - for_each = var.format == "APT" && var.repo_public_repository != null ? [1] : [] - content { - public_repository { - repository_base = var.repository_base - repository_path = var.repository_path - } - } - } - - dynamic "apt_repository" { - for_each = var.format == "APT" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "yum_repository" { - for_each = var.format == "YUM" && var.repo_public_repository != null ? [1] : [] - content { - public_repository { - repository_base = var.repository_base - repository_path = var.repository_path - } - } - } - - dynamic "yum_repository" { - for_each = var.format == "YUM" && var.repo_public_repository == null && var.repo_mirror_url != null ? [1] : [] - content { - custom_repository { - uri = var.repo_mirror_url - } - } - } - - dynamic "common_repository" { - for_each = var.format == "COMMON" ? [1] : [] - content { - uri = var.repo_mirror_url - } - } - - # Only enable upstream credentials if user wants it - dynamic "upstream_credentials" { - for_each = var.use_upstream_credentials ? [1] : [] - content { - username_password_credentials { - username = var.repo_username - password_secret_version = google_secret_manager_secret_version.repo_password_secret_version[0].name - } - } - } - } - } - - depends_on = [ - google_secret_manager_secret.repo_password_secret, - google_secret_manager_secret_version.repo_password_secret_version, - ] -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/metadata.yaml deleted file mode 100644 index 6b68c98a54..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - secretmanager.googleapis.com - - artifactregistry.googleapis.com - - cloudbuild.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/outputs.tf deleted file mode 100644 index 92b6dbb165..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/outputs.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "registry_url" { - description = "The URL of the created artifact registry." - value = "${var.region}-docker.pkg.dev/${var.project_id}/${var.deployment_name}" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/validation.tf b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/validation.tf deleted file mode 100644 index a795060fb7..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/validation.tf +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resource "terraform_data" "input_validation" { - lifecycle { - precondition { - condition = ( - var.repo_password == null || - (var.use_upstream_credentials && var.repo_mode == "REMOTE_REPOSITORY") - ) - error_message = "repo_password may be set only when repo_mode=REMOTE_REPOSITORY and use_upstream_credentials=true." - } - - precondition { - condition = ( - !var.use_upstream_credentials || - var.repo_mode == "REMOTE_REPOSITORY" - ) - error_message = "use_upstream_credentials is allowed only when repo_mode is REMOTE_REPOSITORY." - } - - precondition { - condition = ( - var.repo_mode != "REMOTE_REPOSITORY" || - (var.repo_public_repository != null || var.repo_mirror_url != null) - ) - error_message = "For a REMOTE_REPOSITORY you must set repo_public_repository or repo_mirror_url." - } - - precondition { - condition = ( - !contains(["APT", "YUM"], var.format) || - (var.repository_base != null && var.repository_path != null) - ) - error_message = "APT/YUM formats require repository_base and repository_path." - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/variables.tf deleted file mode 100644 index 9a4eecb921..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/variables.tf +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - description = "Project ID where the artifact registry and secret are created." - type = string -} - -variable "region" { - description = "Region for the artifact registry." - type = string -} - -variable "deployment_name" { - description = "The name of the current deployment." - type = string -} - -variable "labels" { - description = "Labels to add to the artifact registry. Key-value pairs." - type = map(string) - default = {} -} - -variable "repo_password" { - description = "Optional password/API key. If null, one will be randomly generated." - type = string - default = null -} - -variable "user_managed_replication" { - description = <<-DOC - (Optional) A list of objects to enable user-managed replication. - Each object can have: - location = string - kms_key_name = optional(string) - If empty, auto replication is used. - DOC - type = list(object({ - location = string - kms_key_name = optional(string) - })) - default = [] -} - -variable "format" { - description = "Artifact Registry format (e.g., DOCKER)." - type = string - default = "DOCKER" -} - -variable "repo_mode" { - description = "Artifact Registry mode (STANDARD_REPOSITORY, REMOTE_REPOSITORY, etc.)." - type = string - default = "STANDARD_REPOSITORY" - - validation { - condition = can(regex("^(STANDARD_REPOSITORY|REMOTE_REPOSITORY|VIRTUAL_REPOSITORY)$", var.repo_mode)) - error_message = "repo_mode must be one of STANDARD_REPOSITORY, REMOTE_REPOSITORY, or VIRTUAL_REPOSITORY." - } -} - -variable "repo_public_repository" { - description = <<-DOC - For REMOTE_REPOSITORY, name of a known public repo as per the Terraform module - (e.g., DOCKER_HUB) or null for custom repo. - DOC - type = string - default = null - - # To Do: implement validation - # validation { - # condition = ((var.repo_mode != "REMOTE_REPOSITORY" && var.repo_public_repository == null) || (var.repo_mode == "REMOTE_REPOSITORY" && (var.repo_public_repository != null || var.repo_mirror_url != null))) - # error_message = "If repo_mode is REMOTE_REPOSITORY, you must set either repo_public_repository or repo_mirror_url. Otherwise, leave them null." - # } -} - -variable "repo_mirror_url" { - description = "For REMOTE_REPOSITORY, URL for a custom or common mirror." - type = string - default = null -} - -variable "use_upstream_credentials" { - description = <<-DOC - Configure Service Account to use upstream credentials for REMOTE_REPOSITORY: - If true, a username/password is used for the REMOTE_REPOSITORY mirror. - If false (or if repo_password == null), no password is created at all. - Note: Blueprint credentials will be stored in Secrets Manager. - DOC - type = bool - default = false -} - -variable "repo_username" { - description = "Username for external repository." - type = string - default = null -} - -variable "repository_base" { - description = "For APT/YUM public repos, repository_base (e.g., 'DEBIAN', 'UBUNTU')." - type = string - default = null -} - -variable "repository_path" { - description = "For APT/YUM public repos, repository_path (e.g., 'debian/dists/buster')." - type = string - default = null -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/versions.tf deleted file mode 100644 index 392a7131d2..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/container/artifact-registry/versions.tf +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/README.md deleted file mode 100644 index 23bf87398a..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/README.md +++ /dev/null @@ -1,76 +0,0 @@ -## Description - -Creates a BigQuery dataset. - -Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -## Usage -This is a simple usage. - -```yaml - - id: bq-dataset - source: community/modules/database/bigquery-dataset - settings: - dataset_id: my_dataset -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_bigquery_dataset.pbsb](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_dataset) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [dataset\_id](#input\_dataset\_id) | The name of the dataset to be created | `string` | `null` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the dataset. Key-value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [dataset\_id](#output\_dataset\_id) | Name of the dataset that was created. | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/main.tf deleted file mode 100644 index 1a9c4bba60..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/main.tf +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "bigquery-dataset", ghpc_role = "database" }) -} -locals { - dataset_id = var.dataset_id != null ? var.dataset_id : replace("${var.deployment_name}_dataset_${random_id.resource_name_suffix.hex}", "-", "_") -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_bigquery_dataset" "pbsb" { - dataset_id = local.dataset_id - project = var.project_id - labels = local.labels -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml deleted file mode 100644 index 87ff9357e4..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - bigquery.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf deleted file mode 100644 index 9cd8e5df31..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/outputs.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "dataset_id" { - description = "Name of the dataset that was created." - value = google_bigquery_dataset.pbsb.dataset_id -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/variables.tf deleted file mode 100644 index 90c229af6b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/variables.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "dataset_id" { - description = "The name of the dataset to be created" - type = string - default = null -} - -variable "labels" { - description = "Labels to add to the dataset. Key-value pairs." - type = map(string) -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/versions.tf deleted file mode 100644 index 12ddbe842d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-dataset/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/README.md deleted file mode 100644 index ef67cfef01..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/README.md +++ /dev/null @@ -1,87 +0,0 @@ -## Description - -Creates a BigQuery table with a specified schema. - -Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -## Usage - -```yaml -id: bq-table - source: community/modules/database/bigquery-table - use: [bq-dataset] - settings: - table_schema: - ' - [ - { - "name": "id", "type": "STRING" - } - ] - ' -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_bigquery_table.pbsb](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_table) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [dataset\_id](#input\_dataset\_id) | Dataset name to be used to create the new BQ Table | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the tables. Key-value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [table\_id](#input\_table\_id) | Table name to be used to create the new BQ Table | `string` | `null` | no | -| [table\_schema](#input\_table\_schema) | Schema used to create the new BQ Table | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [dataset\_id](#output\_dataset\_id) | ID of BQ dataset | -| [table\_id](#output\_table\_id) | ID of created BQ table | -| [table\_name](#output\_table\_name) | Name of created BQ table | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/main.tf deleted file mode 100644 index 73f3923e00..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/main.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "bigquery-table", ghpc_role = "database" }) -} - -locals { - table_id = var.table_id != null ? var.table_id : replace("${var.deployment_name}_table_${random_id.resource_name_suffix.hex}", "-", "_") -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_bigquery_table" "pbsb" { - deletion_protection = false - project = var.project_id - table_id = local.table_id - dataset_id = var.dataset_id - schema = var.table_schema - labels = local.labels -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/metadata.yaml deleted file mode 100644 index 87ff9357e4..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - bigquery.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/outputs.tf deleted file mode 100644 index 4220ec1390..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/outputs.tf +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "table_name" { - description = "Name of created BQ table" - value = google_bigquery_table.pbsb.friendly_name -} -output "table_id" { - description = "ID of created BQ table" - value = google_bigquery_table.pbsb.table_id -} -output "dataset_id" { - description = "ID of BQ dataset" - value = google_bigquery_table.pbsb.dataset_id -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/variables.tf deleted file mode 100644 index ec474b4e64..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/variables.tf +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "labels" { - description = "Labels to add to the tables. Key-value pairs." - type = map(string) -} - -variable "table_id" { - description = "Table name to be used to create the new BQ Table" - type = string - default = null -} - -variable "dataset_id" { - description = "Dataset name to be used to create the new BQ Table" - type = string -} - -variable "table_schema" { - description = "Schema used to create the new BQ Table" - type = string -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/versions.tf deleted file mode 100644 index 12ddbe842d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/database/bigquery-table/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md deleted file mode 100644 index 08364c175b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/README.md +++ /dev/null @@ -1,107 +0,0 @@ -## Description - -terraform-google-sql makes it easy to create a Google CloudSQL instance and -implement high availability settings. This module is meant for use with -Terraform 0.13+ and tested using Terraform 1.0+. - -The cloudsql created here is used to integrate with the slurm cluster to enable -accounting data storage. - -### Example - -```yaml -- id: cloudsql - source: community/modules/database/slurm-cloudsql-federation - use: [network] - settings: - sql_instance_name: slurm-sql6-demo - tier: "db-f1-micro" -``` - -This creates a cloud sql instance, including a database, user that would allow -the slurm cluster to use as an external DB. In addition, it will allow BigQuery -to run federated query through it. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.0 | -| [google](#requirement\_google) | >= 3.83 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_bigquery_connection.connection](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_connection) | resource | -| [google_compute_address.psc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | -| [google_compute_forwarding_rule.psc_consumer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_forwarding_rule) | resource | -| [google_sql_database.database](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database) | resource | -| [google_sql_database_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database_instance) | resource | -| [google_sql_user.users](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_user) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [random_password.password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [authorized\_networks](#input\_authorized\_networks) | IP address ranges as authorized networks of the Cloud SQL for MySQL instances | `list(string)` | `[]` | no | -| [data\_cache\_enabled](#input\_data\_cache\_enabled) | Whether data cache is enabled for the instance. Can be used with ENTERPRISE\_PLUS edition. | `bool` | `false` | no | -| [database\_flags](#input\_database\_flags) | Database flags to set on instance. | `map(string)` | `{}` | no | -| [database\_version](#input\_database\_version) | The version of the database to be created. | `string` | `"MYSQL_8_0"` | no | -| [deletion\_protection](#input\_deletion\_protection) | Whether or not to allow Terraform to destroy the instance. | `string` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [disk\_autoresize](#input\_disk\_autoresize) | Set to false to disable automatic disk grow. | `bool` | `true` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of the database disk in GiB. | `number` | `null` | no | -| [edition](#input\_edition) | value | `string` | `"ENTERPRISE"` | no | -| [enable\_backups](#input\_enable\_backups) | Set true to enable backups | `bool` | `false` | no | -| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is going to be created in.:
`projects//global/networks/`" | `string` | n/a | yes | -| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection, used only as dependency for Cloud SQL creation. | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [query\_insights](#input\_query\_insights) | Query insights configuration. | 
object({
    enabled                 = optional(bool, false)
    query_plans_per_minute  = optional(number)
    query_string_length     = optional(number)
    record_application_tags = optional(bool)
    record_client_address   = optional(bool)
  })
| `{}` | no | -| [region](#input\_region) | The region where SQL instance will be configured | `string` | n/a | yes | -| [sql\_instance\_name](#input\_sql\_instance\_name) | name given to the sql instance for ease of identificaion | `string` | n/a | yes | -| [sql\_password](#input\_sql\_password) | Password for the SQL database. | `any` | `null` | no | -| [sql\_username](#input\_sql\_username) | Username for the SQL database | `string` | `"slurm"` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Self link of the network where Cloud SQL instance PSC endpoint will be created | `string` | `null` | no | -| [tier](#input\_tier) | The machine type to use for the SQL instance | `string` | n/a | yes | -| [use\_psc\_connection](#input\_use\_psc\_connection) | Create Private Service Connection instead of using Private Service Access peering | `bool` | `false` | no | -| [user\_managed\_replication](#input\_user\_managed\_replication) | Replication parameters that will be used for defined secrets | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [cloudsql](#output\_cloudsql) | Describes the cloudsql instance. | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf deleted file mode 100644 index 9b518a1b5f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/main.tf +++ /dev/null @@ -1,165 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "slurm-cloudsql-federation", ghpc_role = "database" }) -} - -locals { - user_managed_replication = var.user_managed_replication -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "random_password" "password" { - length = 12 - special = false -} - -locals { - sql_instance_name = var.sql_instance_name == null ? "${var.deployment_name}-sql-${random_id.resource_name_suffix.hex}" : var.sql_instance_name - sql_password = var.sql_password == null ? random_password.password.result : var.sql_password -} - - -resource "google_sql_database_instance" "instance" { - project = var.project_id - depends_on = [var.private_vpc_connection_peering] - name = local.sql_instance_name - region = var.region - deletion_protection = var.deletion_protection - database_version = var.database_version - - settings { - disk_size = var.disk_size_gb - disk_autoresize = var.disk_autoresize - edition = var.edition - tier = var.tier - user_labels = local.labels - - dynamic "data_cache_config" { - for_each = var.edition == "ENTERPRISE_PLUS" ? [""] : [] - content { - data_cache_enabled = var.data_cache_enabled - } - } - - dynamic "database_flags" { - for_each = var.database_flags - content { - name = database_flags.key - value = database_flags.value - } - } - - insights_config { - query_insights_enabled = var.query_insights.enabled - query_plans_per_minute = var.query_insights.query_plans_per_minute - query_string_length = var.query_insights.query_string_length - record_application_tags = var.query_insights.record_application_tags - record_client_address = var.query_insights.record_client_address - } - - ip_configuration { - ipv4_enabled = false - private_network = var.use_psc_connection ? null : var.network_id - enable_private_path_for_google_cloud_services = true - - dynamic "authorized_networks" { - for_each = var.use_psc_connection ? [] : var.authorized_networks - iterator = ip_range - - content { - value = ip_range.value - } - } - dynamic "psc_config" { - for_each = var.use_psc_connection ? [""] : [] - content { - psc_enabled = true - allowed_consumer_projects = [var.project_id] - } - } - } - - backup_configuration { - enabled = var.enable_backups - # to allow easy switching between ENTERPRISE and ENTERPRISE_PLUS - transaction_log_retention_days = 7 - } - } - lifecycle { - precondition { - condition = var.disk_autoresize && var.disk_size_gb == null || !var.disk_autoresize - error_message = "If setting disk_size_gb set disk_autorize to false to prevent re-provisioning of the instance after disk auto-expansion." - } - } -} - - - -resource "google_compute_address" "psc" { - count = var.use_psc_connection ? 1 : 0 - project = var.project_id - name = local.sql_instance_name - address_type = "INTERNAL" - region = var.region - subnetwork = var.subnetwork_self_link - labels = local.labels -} - -resource "google_compute_forwarding_rule" "psc_consumer" { - count = var.use_psc_connection ? 1 : 0 - name = local.sql_instance_name - project = var.project_id - region = var.region - subnetwork = var.subnetwork_self_link - ip_address = google_compute_address.psc[0].self_link - load_balancing_scheme = "" - recreate_closed_psc = true - target = google_sql_database_instance.instance.psc_service_attachment_link -} - -resource "google_sql_database" "database" { - project = var.project_id - name = "slurm_accounting" - instance = google_sql_database_instance.instance.name -} - -resource "google_sql_user" "users" { - project = var.project_id - name = var.sql_username - instance = google_sql_database_instance.instance.name - password = local.sql_password -} - -resource "google_bigquery_connection" "connection" { - provider = google - project = var.project_id - location = var.region - cloud_sql { - instance_id = google_sql_database_instance.instance.connection_name - database = google_sql_database.database.name - type = "MYSQL" - credential { - username = google_sql_user.users.name - password = google_sql_user.users.password - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml deleted file mode 100644 index fc0cae0859..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - bigqueryconnection.googleapis.com - - sqladmin.googleapis.com - - servicenetworking.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf deleted file mode 100644 index 0d05221cd8..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/outputs.tf +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "cloudsql" { - description = "Describes the cloudsql instance." - sensitive = true - value = { - server_ip = var.use_psc_connection ? google_compute_address.psc[0].address : google_sql_database_instance.instance.ip_address[0].ip_address - user = google_sql_user.users.name - password = google_sql_user.users.password - db_name = google_sql_database.database.name - user_managed_replication = local.user_managed_replication - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf deleted file mode 100644 index a2f150419e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/variables.tf +++ /dev/null @@ -1,173 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "authorized_networks" { - description = "IP address ranges as authorized networks of the Cloud SQL for MySQL instances" - type = list(string) - default = [] - nullable = false -} - -variable "database_version" { - description = "The version of the database to be created." - type = string - default = "MYSQL_8_0" - validation { - condition = contains(["MYSQL_5_7", "MYSQL_8_0", "MYSQL_8_4"], var.database_version) - error_message = "The database version must be either MYSQL_5_7, MYSQL_8_0 or MYSQL_8_4." - } -} - -variable "data_cache_enabled" { - description = "Whether data cache is enabled for the instance. Can be used with ENTERPRISE_PLUS edition." - type = bool - default = false -} - -variable "database_flags" { - description = "Database flags to set on instance." - type = map(string) - default = {} - nullable = false -} - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "disk_autoresize" { - description = "Set to false to disable automatic disk grow." - type = bool - default = true -} - -variable "disk_size_gb" { - description = "Size of the database disk in GiB." - type = number - default = null -} - -variable "edition" { - description = "value" - type = string - validation { - condition = contains(["ENTERPRISE", "ENTERPRISE_PLUS"], var.edition) - error_message = "The database edition must be either ENTERPRISE or ENTERPRISE_PLUS" - } - default = "ENTERPRISE" -} - -variable "enable_backups" { - description = "Set true to enable backups" - type = bool - default = false -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "query_insights" { - description = "Query insights configuration." - nullable = false - default = {} - type = object({ - enabled = optional(bool, false) - query_plans_per_minute = optional(number) - query_string_length = optional(number) - record_application_tags = optional(bool) - record_client_address = optional(bool) - }) -} - -variable "region" { - description = "The region where SQL instance will be configured" - type = string -} - -variable "tier" { - description = "The machine type to use for the SQL instance" - type = string -} - -variable "sql_instance_name" { - description = "name given to the sql instance for ease of identificaion" - type = string -} - -variable "deletion_protection" { - description = "Whether or not to allow Terraform to destroy the instance." - type = string - default = false -} - -variable "labels" { - description = "Labels to add to the instances. Key-value pairs." - type = map(string) -} - -variable "sql_username" { - description = "Username for the SQL database" - type = string - default = "slurm" -} - -variable "sql_password" { - description = "Password for the SQL database." - type = any - default = null -} - -variable "network_id" { - description = <<-EOT - The ID of the GCE VPC network to which the instance is going to be created in.: - `projects//global/networks/`" - EOT - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "private_vpc_connection_peering" { - description = "The name of the VPC Network peering connection, used only as dependency for Cloud SQL creation." - type = string - default = null -} - -variable "subnetwork_self_link" { - description = "Self link of the network where Cloud SQL instance PSC endpoint will be created" - type = string - default = null -} - -variable "user_managed_replication" { - type = list(object({ - location = string - kms_key_name = optional(string) - })) - description = "Replication parameters that will be used for defined secrets" - default = [] -} - -variable "use_psc_connection" { - description = "Create Private Service Connection instead of using Private Service Access peering" - type = bool - default = false -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf deleted file mode 100644 index 922e383a69..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/database/slurm-cloudsql-federation/versions.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.57.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.57.0" - } - - required_version = ">= 0.13.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md deleted file mode 100644 index d39a58afe1..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/README.md +++ /dev/null @@ -1,158 +0,0 @@ -> [!WARNING] -> This module is deprecated and will be removed on July 1, 2025. The -> recommended replacement is the -> [GCP Managed Lustre module](../../../../modules/file-system/managed-lustre/README.md) - -## Description -This module creates a DDN EXAScaler Cloud Lustre file system using code based on DDN's -[exascaler-cloud-terraform](https://github.com/DDNStorage/exascaler-cloud-terraform/tree/scripts/2.2.2/gcp) (`scripts/2.2.2` is last release with GCP-specific module). - -More information about the architecture can be found at -[Overview of Lustre and EXAScaler Cloud][architecture]. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). - -> **Warning**: This file system has a license cost as described in the pricing -> section of the [DDN EXAScaler Cloud Marketplace Solution][marketplace]. -> -> **Note**: By default security.public_key is set to `null`, therefore the -> admin user is not created. To ensure the admin user is created, provide a -> public key via the security setting. -> -> **Note**: This module's instances require access to Google APIs and -> therefore, instances must have public IP address or it must be used in a -> subnetwork where [Private Google Access][private-google-access] is enabled. - -[private-google-access]: https://cloud.google.com/vpc/docs/configure-private-google-access -[marketplace]: https://console.developers.google.com/marketplace/product/ddnstorage/exascaler-cloud -[architecture]: https://cloud.google.com/architecture/parallel-file-systems-for-hpc#overview_of_lustre_and_exascaler_cloud - -## Mounting - -To mount the DDN EXAScaler Lustre file system you must first install the DDN -Lustre client and then call the proper `mount` command. - -Both of these steps are automatically handled with the use of the `use` command -in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in -the network storage doc for a complete list of supported modules. -the [hpc-enterprise-slurm.yaml](../../../../examples/hpc-enterprise-slurm.yaml) for an -example of using this module with Slurm. - -If mounting is not automatically handled as described above, the DDN-EXAScaler -module outputs runners that can be used with the startup-script module to -install the client and mount the file system. See the following example: - -```yaml - # This file system has an associated license cost. - # https://console.developers.google.com/marketplace/product/ddnstorage/exascaler-cloud - - id: lustrefs - source: community/modules/file-system/DDN-EXAScaler - use: [network1] - settings: {local_mount: /scratch} - - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: - - $(lustrefs.install_ddn_lustre_client_runner) - - $(lustrefs.mount_runner) - -``` - -See [additional documentation][ddn-install-docs] from DDN EXAScaler. - -[ddn-install-docs]: https://github.com/DDNStorage/exascaler-cloud-terraform/tree/scripts/2.2.2/gcp#install-new-exascaler-cloud-clients -[matrix]: ../../../../docs/network_storage.md#compatibility-matrix - -## Support - -EXAScaler Cloud includes self-help support with access to publicly available -documents and videos. Premium support includes 24x7x365 access to DDN's experts, -along with support community access, automated notifications of updates and -other premium support features. For more information, visit -[EXAscaler Cloud on GCP][exa-gcp]. - -[exa-gcp]: https://console.cloud.google.com/marketplace/product/ddnstorage/exascaler-cloud - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [ddn\_exascaler](#module\_ddn\_exascaler) | github.com/DDNStorage/exascaler-cloud-terraform//gcp | a3355d50deebe45c0556b45bd599059b7c06988d | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [boot](#input\_boot) | Boot disk properties | 
object({
    disk_type   = string
    auto_delete = bool
    script_url  = string
  })
| 
{
  "auto_delete": true,
  "disk_type": "pd-standard",
  "script_url": null
}
| no | -| [cls](#input\_cls) | Compute client properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 0,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-2",
  "public_ip": true
}
| no | -| [clt](#input\_clt) | Compute client target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 0,
  "disk_size": 256,
  "disk_type": "pd-standard"
}
| no | -| [fsname](#input\_fsname) | EXAScaler filesystem name, only alphanumeric characters are allowed, and the value must be 1-8 characters long | `string` | `"exacloud"` | no | -| [image](#input\_image) | DEPRECATED: Source image properties | `any` | `null` | no | -| [instance\_image](#input\_instance\_image) | Source image properties

Expected Fields:
name: Unavailable with this module.
family: The image family to use.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "exascaler-cloud-6-2-rocky-linux-8-optimized-gcp",
  "project": "ddn-public"
}
| no | -| [labels](#input\_labels) | Labels to add to EXAScaler Cloud deployment. Key-value pairs. | `map(string)` | `{}` | no | -| [local\_mount](#input\_local\_mount) | Mountpoint (at the client instances) for this EXAScaler system | `string` | `"/shared"` | no | -| [mds](#input\_mds) | Metadata server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 1,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-32",
  "public_ip": true
}
| no | -| [mdt](#input\_mdt) | Metadata target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 3500,
  "disk_type": "pd-ssd"
}
| no | -| [mgs](#input\_mgs) | Management server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 1,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-32",
  "public_ip": true
}
| no | -| [mgt](#input\_mgt) | Management target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 128,
  "disk_type": "pd-standard"
}
| no | -| [mnt](#input\_mnt) | Monitoring target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 128,
  "disk_type": "pd-standard"
}
| no | -| [network\_properties](#input\_network\_properties) | Network options. 'network\_self\_link' or 'network\_properties' must be provided. | 
object({
    routing = string
    tier    = string
    id      = string
    auto    = bool
    mtu     = number
    new     = bool
    nat     = bool
  })
| `null` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self-link of the VPC network to where the system is connected. Ignored if 'network\_properties' is provided. 'network\_self\_link' or 'network\_properties' must be provided. | `string` | `null` | no | -| [oss](#input\_oss) | Object Storage server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 3,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-16",
  "public_ip": true
}
| no | -| [ost](#input\_ost) | Object Storage target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 3500,
  "disk_type": "pd-ssd"
}
| no | -| [prefix](#input\_prefix) | EXAScaler Cloud deployment prefix (`null` defaults to 'exascaler-cloud') | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Compute Platform project that will host the EXAScaler filesystem | `string` | n/a | yes | -| [security](#input\_security) | Security options | 
object({
    admin              = string
    public_key         = string
    block_project_keys = bool
    enable_os_login    = bool
    enable_local       = bool
    enable_ssh         = bool
    enable_http        = bool
    ssh_source_ranges  = list(string)
    http_source_ranges = list(string)
  })
| 
{
  "admin": "stack",
  "block_project_keys": false,
  "enable_http": false,
  "enable_local": false,
  "enable_os_login": true,
  "enable_ssh": false,
  "http_source_ranges": [
    "0.0.0.0/0"
  ],
  "public_key": null,
  "ssh_source_ranges": [
    "0.0.0.0/0"
  ]
}
| no | -| [service\_account](#input\_service\_account) | Service account name used by deploy application | 
object({
    new   = bool
    email = string
  })
| 
{
  "email": null,
  "new": false
}
| no | -| [subnetwork\_address](#input\_subnetwork\_address) | The IP range of internal addresses for the subnetwork. Ignored if 'subnetwork\_properties' is provided. | `string` | `null` | no | -| [subnetwork\_properties](#input\_subnetwork\_properties) | Subnetwork properties. 'subnetwork\_self\_link' or 'subnetwork\_properties' must be provided. | 
object({
    address = string
    private = bool
    id      = string
    new     = bool
  })
| `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self-link of the VPC subnetwork to where the system is connected. Ignored if 'subnetwork\_properties' is provided. 'subnetwork\_self\_link' or 'subnetwork\_properties' must be provided. | `string` | `null` | no | -| [waiter](#input\_waiter) | Waiter to check progress and result for deployment. | `string` | `null` | no | -| [zone](#input\_zone) | Compute Platform zone where the servers will be located | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [client\_config\_script](#output\_client\_config\_script) | Script that will install DDN EXAScaler lustre client. The machine running this script must be on the same network & subnet as the EXAScaler. | -| [http\_console](#output\_http\_console) | HTTP address to access the system web console. | -| [install\_ddn\_lustre\_client\_runner](#output\_install\_ddn\_lustre\_client\_runner) | Runner that encapsulates the `client_config_script` output on this module. | -| [mount\_command](#output\_mount\_command) | Command to mount the file system. `client_config_script` must be run first. | -| [mount\_runner](#output\_mount\_runner) | Runner to mount the DDN EXAScaler Lustre file system | -| [network\_storage](#output\_network\_storage) | Describes a EXAScaler system to be mounted by other systems. | -| [private\_addresses](#output\_private\_addresses) | Private IP addresses for all instances. | -| [ssh\_console](#output\_ssh\_console) | Instructions to ssh into the instances. | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf deleted file mode 100644 index 6a2fc4b702..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/main.tf +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# WARNING -# This module is deprecated and will be removed on July 1, 2025 -# The recommended replacement is the Managed Lustre module -# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "ddn-exascaler", ghpc_role = "file-system" }) -} - -locals { - - network_id = var.network_self_link != null ? regex("https://www.googleapis.com/compute/v\\d/(.*)", var.network_self_link)[0] : null - named_net = { - routing = "REGIONAL" - tier = "STANDARD" - id = local.network_id - auto = false - mtu = 1500 - new = false - nat = false - } - - subnetwork_id = var.subnetwork_self_link != null ? regex("https://www.googleapis.com/compute/v\\d/(.*)", var.subnetwork_self_link)[0] : null - named_subnet = { - address = var.subnetwork_address - private = true - id = local.subnetwork_id - new = false - } -} - -module "ddn_exascaler" { - source = "github.com/DDNStorage/exascaler-cloud-terraform//gcp?ref=a3355d50deebe45c0556b45bd599059b7c06988d" - fsname = var.fsname - zone = var.zone - project = var.project_id - prefix = var.prefix - labels = local.labels - security = var.security - service_account = var.service_account - waiter = var.waiter - network = var.network_properties == null ? local.named_net : var.network_properties - subnetwork = var.subnetwork_properties == null ? local.named_subnet : var.subnetwork_properties - boot = var.boot - image = var.instance_image - mgs = var.mgs - mgt = var.mgt - mnt = var.mnt - mds = var.mds - mdt = var.mdt - oss = var.oss - ost = var.ost - cls = var.cls - clt = var.clt -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml deleted file mode 100644 index b995bd4358..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/metadata.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - deploymentmanager.googleapis.com - - iam.googleapis.com - - runtimeconfig.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf deleted file mode 100644 index 2e9ae732ae..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/outputs.tf +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# WARNING -# This module is deprecated and will be removed on July 1, 2025 -# The recommended replacement is the Managed Lustre module -# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre - -output "private_addresses" { - description = "Private IP addresses for all instances." - value = module.ddn_exascaler.private_addresses -} - -output "ssh_console" { - description = "Instructions to ssh into the instances." - value = module.ddn_exascaler.ssh_console -} - -output "client_config_script" { - description = "Script that will install DDN EXAScaler lustre client. The machine running this script must be on the same network & subnet as the EXAScaler." - value = module.ddn_exascaler.client_config -} - -output "install_ddn_lustre_client_runner" { - description = "Runner that encapsulates the `client_config_script` output on this module." - value = local.client_install_runner -} - -locals { - client_install_runner = { - "type" = "shell" - "content" = module.ddn_exascaler.client_config - "destination" = "install_ddn_lustre_client.sh" - } - - # Mount command provided by DDN does not support custom local mount - split_mount_cmd = split(" ", module.ddn_exascaler.mount_command) - split_mount_cmd_wo_mountpoint = slice(local.split_mount_cmd, 0, length(local.split_mount_cmd) - 1) - mount_cmd = "${join(" ", local.split_mount_cmd_wo_mountpoint)} ${var.local_mount}" - mount_cmd_w_mkdir = "mkdir -p ${var.local_mount} && ${local.mount_cmd}" - mount_runner = { - "type" = "shell" - "content" = local.mount_cmd_w_mkdir - "destination" = "mount-ddn-lustre.sh" - } -} - -output "mount_command" { - description = "Command to mount the file system. `client_config_script` must be run first." - value = local.mount_cmd_w_mkdir -} - -output "mount_runner" { - description = "Runner to mount the DDN EXAScaler Lustre file system" - value = local.mount_runner -} - -output "http_console" { - description = "HTTP address to access the system web console." - value = module.ddn_exascaler.http_console -} - -output "network_storage" { - description = "Describes a EXAScaler system to be mounted by other systems." - value = { - server_ip = split(":", split(" ", module.ddn_exascaler.mount_command)[3])[0] - remote_mount = length(regexall("^/.*", var.fsname)) > 0 ? var.fsname : format("/%s", var.fsname) - local_mount = var.local_mount != null ? var.local_mount : format("/mnt/%s", var.fsname) - fs_type = "lustre" - mount_options = "" - client_install_runner = local.client_install_runner - mount_runner = local.mount_runner - } - depends_on = [ - module.ddn_exascaler - ] -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf deleted file mode 100644 index 68bcc8a8ba..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/variables.tf +++ /dev/null @@ -1,502 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# WARNING -# This module is deprecated and will be removed on July 1, 2025 -# The recommended replacement is the Managed Lustre module -# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre - -# EXAScaler filesystem name -# only alphanumeric characters are allowed, -# and the value must be 1-8 characters long -variable "fsname" { - description = "EXAScaler filesystem name, only alphanumeric characters are allowed, and the value must be 1-8 characters long" - type = string - default = "exacloud" -} - -# Project ID to manage resources -# https://cloud.google.com/resource-manager/docs/creating-managing-projects -variable "project_id" { - description = "Compute Platform project that will host the EXAScaler filesystem" - type = string -} - -# Zone name to manage resources -# https://cloud.google.com/compute/docs/regions-zones -variable "zone" { - description = "Compute Platform zone where the servers will be located" - type = string -} - -# Service account name used by deploy application -# https://cloud.google.com/iam/docs/service-accounts -# new: create a new custom service account or use an existing one: true or false -# email: existing service account email address, will be using if new is false -# set email = null to use the default compute service account -variable "service_account" { - description = "Service account name used by deploy application" - type = object({ - new = bool - email = string - }) - default = { - new = false - email = null - } -} - -# Waiter to check progress and result for deployment. -# To use Google Deployment Manager: -# waiter = "deploymentmanager" -# To use generic Google Cloud SDK command line: -# waiter = "sdk" -# If you donā€™t want to wait until the deployment is complete: -# waiter = null -# https://cloud.google.com/deployment-manager/runtime-configurator/creating-a-waiter -variable "waiter" { - description = "Waiter to check progress and result for deployment." - type = string - default = null -} - -# Security options -# admin: optional user name for remote SSH access -# Set admin = null to disable creation admin user -# public_key: path to the SSH public key on the local host -# Set public_key = null to disable creation admin user -# block_project_keys: true or false -# Block project-wide public SSH keys if you want to restrict -# deployment to only user with deployment-level public SSH key. -# https://cloud.google.com/compute/docs/instances/adding-removing-ssh-keys -# enable_os_login: true or false -# Enable or disable OS Login feature. -# Please note, enabling this option disables other security options: -# admin, public_key and block_project_keys. -# https://cloud.google.com/compute/docs/instances/managing-instance-access#enable_oslogin -# enable_local: true or false, enable or disable firewall rules for local access -# enable_ssh: true or false, enable or disable remote SSH access -# ssh_source_ranges: source IP ranges for remote SSH access in CIDR notation -# enable_http: true or false, enable or disable remote HTTP access -# http_source_ranges: source IP ranges for remote HTTP access in CIDR notation -variable "security" { - description = "Security options" - type = object({ - admin = string - public_key = string - block_project_keys = bool - enable_os_login = bool - enable_local = bool - enable_ssh = bool - enable_http = bool - ssh_source_ranges = list(string) - http_source_ranges = list(string) - }) - - default = { - admin = "stack" - public_key = null - block_project_keys = false - enable_os_login = true - enable_local = false - enable_ssh = false - enable_http = false - ssh_source_ranges = [ - "0.0.0.0/0" - ] - http_source_ranges = [ - "0.0.0.0/0" - ] - } -} - -variable "network_self_link" { - description = "The self-link of the VPC network to where the system is connected. Ignored if 'network_properties' is provided. 'network_self_link' or 'network_properties' must be provided." - type = string - default = null -} - -# Network properties -# https://cloud.google.com/vpc/docs/vpc -# routing: network-wide routing mode: REGIONAL or GLOBAL -# tier: networking tier for VM interfaces: STANDARD or PREMIUM -# id: existing network id, will be using if new is false -# auto: create subnets in each region automatically: false or true -# mtu: maximum transmission unit in bytes: 1460 - 1500 -# new: create a new network or use an existing one: true or false -# nat: allow instances without external IP to communicate with the outside world: true or false -variable "network_properties" { - description = "Network options. 'network_self_link' or 'network_properties' must be provided." - type = object({ - routing = string - tier = string - id = string - auto = bool - mtu = number - new = bool - nat = bool - }) - - default = null -} - -variable "subnetwork_self_link" { - description = "The self-link of the VPC subnetwork to where the system is connected. Ignored if 'subnetwork_properties' is provided. 'subnetwork_self_link' or 'subnetwork_properties' must be provided." - type = string - default = null -} - -variable "subnetwork_address" { - description = "The IP range of internal addresses for the subnetwork. Ignored if 'subnetwork_properties' is provided." - type = string - default = null -} - -# Subnetwork properties -# https://cloud.google.com/vpc/docs/vpc -# address: IP range of internal addresses for a new subnetwork -# private: when enabled VMs in this subnetwork without external -# IP addresses can access Google APIs and services by using -# Private Google Access: true or false -# https://cloud.google.com/vpc/docs/private-access-options -# id: existing subnetwork id, will be using if new is false -# new: create a new subnetwork or use an existing one: true or false -variable "subnetwork_properties" { - description = "Subnetwork properties. 'subnetwork_self_link' or 'subnetwork_properties' must be provided." - type = object({ - address = string - private = bool - id = string - new = bool - }) - default = null -} -# Boot disk properties -# disk_type: pd-standard, pd-ssd or pd-balanced -# auto_delete: true or false -# whether the disk will be auto-deleted when the instance is deleted -variable "boot" { - description = "Boot disk properties" - type = object({ - disk_type = string - auto_delete = bool - script_url = string - }) - default = { - disk_type = "pd-standard" - auto_delete = true - script_url = null - } -} - -# Source image properties -# project: project name -# family: image family name -# name: !!DEPRECATED!! - image name -# tflint-ignore: terraform_unused_declarations -variable "image" { - description = "DEPRECATED: Source image properties" - type = any - # Omitting type checking so validation can provide more useful error message - # type = object({ - # project = string - # family = string - # }) - default = null - - validation { - condition = var.image == null - error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." - } -} - -variable "instance_image" { - description = <<-EOD - Source image properties - - Expected Fields: - name: Unavailable with this module. - family: The image family to use. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "ddn-public" - family = "exascaler-cloud-6-2-rocky-linux-8-optimized-gcp" - } - - validation { - condition = !can(coalesce(var.instance_image.name)) - error_message = "In var.instance_image, the \"name\" field is not used, please use the \"family\" setting." - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, the \"family\" field must be a string set to the image family." - } -} - -# Management server properties -# node_type: type of management server -# https://cloud.google.com/compute/docs/machine-types -# node_cpu: CPU family -# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform -# nic_type: type of network connectivity, GVNIC or VIRTIO_NET -# https://cloud.google.com/compute/docs/networking/using-gvnic -# public_ip: assign an external IP address, true or false -# node_count: number of management servers -variable "mgs" { - description = "Management server properties" - type = object({ - node_type = string - node_cpu = string - nic_type = string - node_count = number - public_ip = bool - }) - default = { - node_type = "n2-standard-32" - node_cpu = "Intel Cascade Lake" - nic_type = "GVNIC" - public_ip = true - node_count = 1 - } -} - -# Management target properties -# https://cloud.google.com/compute/docs/disks -# disk_bus: type of management target interface, SCSI or NVME (NVME is for scratch disks only) -# disk_type: type of management target, pd-standard, pd-ssd, pd-balanced or scratch -# disk_size: size of management target in GB (scratch disk size must be exactly 375) -# disk_count: number of management targets -# disk_raid: create striped management target, true or false -variable "mgt" { - description = "Management target properties" - type = object({ - disk_bus = string - disk_type = string - disk_size = number - disk_count = number - disk_raid = bool - }) - default = { - disk_bus = "SCSI" - disk_type = "pd-standard" - disk_size = 128 - disk_count = 1 - disk_raid = false - } -} - - -# Monitoring target properties -# https://cloud.google.com/compute/docs/disks -# disk_bus: type of monitoring target interface, SCSI or NVME (NVME is for scratch disks only) -# disk_type: type of monitoring target, pd-standard, pd-ssd, pd-balanced or scratch -# disk_size: size of monitoring target in GB (scratch disk size must be exactly 375) -# disk_count: number of monitoring targets -# disk_raid: create striped monitoring target, true or false -variable "mnt" { - description = "Monitoring target properties" - type = object({ - disk_bus = string - disk_type = string - disk_size = number - disk_count = number - disk_raid = bool - }) - default = { - disk_bus = "SCSI" - disk_type = "pd-standard" - disk_size = 128 - disk_count = 1 - disk_raid = false - } -} - -# Metadata server properties -# node_type: type of metadata server -# https://cloud.google.com/compute/docs/machine-types -# node_cpu: CPU family -# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform -# nic_type: type of network connectivity, GVNIC or VIRTIO_NET -# https://cloud.google.com/compute/docs/networking/using-gvnic -# public_ip: assign an external IP address, true or false -# node_count: number of metadata servers -variable "mds" { - description = "Metadata server properties" - type = object({ - node_type = string - node_cpu = string - nic_type = string - node_count = number - public_ip = bool - }) - default = { - node_type = "n2-standard-32" - node_cpu = "Intel Cascade Lake" - nic_type = "GVNIC" - public_ip = true - node_count = 1 - } -} - -# Metadata target properties -# https://cloud.google.com/compute/docs/disks -# disk_bus: type of metadata target interface, SCSI or NVME (NVME is for scratch disks only) -# disk_type: type of metadata target, pd-standard, pd-ssd, pd-balanced or scratch -# disk_size: size of metadata target in GB (scratch disk size must be exactly 375) -# disk_count: number of metadata targets -# disk_raid: create striped metadata target, true or false -variable "mdt" { - description = "Metadata target properties" - type = object({ - disk_bus = string - disk_type = string - disk_size = number - disk_count = number - disk_raid = bool - }) - default = { - disk_bus = "SCSI" - disk_type = "pd-ssd" - disk_size = 3500 - disk_count = 1 - disk_raid = false - } -} - -# Object Storage server properties -# node_type: type of storage server -# https://cloud.google.com/compute/docs/machine-types -# node_cpu: CPU family -# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform -# nic_type: type of network connectivity, GVNIC or VIRTIO_NET -# https://cloud.google.com/compute/docs/networking/using-gvnic -# public_ip: assign an external IP address, true or false -# node_count: number of storage servers -variable "oss" { - description = "Object Storage server properties" - type = object({ - node_type = string - node_cpu = string - nic_type = string - node_count = number - public_ip = bool - }) - default = { - node_type = "n2-standard-16" - node_cpu = "Intel Cascade Lake" - nic_type = "GVNIC" - public_ip = true - node_count = 3 - } -} - -# Object Storage target properties -# https://cloud.google.com/compute/docs/disks -# disk_bus: type of storage target interface, SCSI or NVME (NVME is for scratch disks only) -# disk_type: type of storage target, pd-standard, pd-ssd, pd-balanced or scratch -# disk_size: size of storage target in GB (scratch disk size must be exactly 375) -# disk_count: number of storage targets -# disk_raid: create striped storage target, true or false -variable "ost" { - description = "Object Storage target properties" - type = object({ - disk_bus = string - disk_type = string - disk_size = number - disk_count = number - disk_raid = bool - }) - default = { - disk_bus = "SCSI" - disk_type = "pd-ssd" - disk_size = 3500 - disk_count = 1 - disk_raid = false - } -} - -# Compute client properties -# node_type: type of compute client -# https://cloud.google.com/compute/docs/machine-types -# node_cpu: CPU family -# https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform -# nic_type: type of network connectivity, GVNIC or VIRTIO_NET -# https://cloud.google.com/compute/docs/networking/using-gvnic -# public_ip: assign an external IP address, true or false -# node_count: number of compute clients -variable "cls" { - description = "Compute client properties" - type = object({ - node_type = string - node_cpu = string - nic_type = string - node_count = number - public_ip = bool - }) - default = { - node_type = "n2-standard-2" - node_cpu = "Intel Cascade Lake" - nic_type = "GVNIC" - public_ip = true - node_count = 0 - } -} -# Compute client target properties -# https://cloud.google.com/compute/docs/disks -# disk_bus: type of compute target interface, SCSI or NVME (NVME is for scratch disks only) -# disk_type: type of compute target, pd-standard, pd-ssd, pd-balanced or scratch -# disk_size: size of compute target in GB (scratch disk size must be exactly 375) -# disk_count: number of compute targets -variable "clt" { - description = "Compute client target properties" - type = object({ - disk_bus = string - disk_type = string - disk_size = number - disk_count = number - }) - default = { - disk_bus = "SCSI" - disk_type = "pd-standard" - disk_size = 256 - disk_count = 0 - } -} -variable "local_mount" { - description = "Mountpoint (at the client instances) for this EXAScaler system" - type = string - default = "/shared" -} - -variable "prefix" { - description = "EXAScaler Cloud deployment prefix (`null` defaults to 'exascaler-cloud')" - type = string - default = null -} - -variable "labels" { - description = "Labels to add to EXAScaler Cloud deployment. Key-value pairs." - type = map(string) - default = {} -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf deleted file mode 100644 index 2981b4dd75..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/DDN-EXAScaler/versions.tf +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -# WARNING -# This module is deprecated and will be removed on July 1, 2025 -# The recommended replacement is the Managed Lustre module -# https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/modules/file-system/managed-lustre - -terraform { - required_version = ">= 0.13.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/Intel-DAOS/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/Intel-DAOS/README.md deleted file mode 100644 index 04db0acb8c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/Intel-DAOS/README.md +++ /dev/null @@ -1 +0,0 @@ -> **_NOTE:_** Cluster Toolkit is dropping support for the external [Google Cloud DAOS](https://github.com/daos-stack/google-cloud-daos/tree/main) repository. The DAOS example blueprints (`hpc-slurm-daos.yaml` and `pfs-daos.yaml`) have been removed from the Cluster Toolkit. We recommend migrating to the first-party [Parallelstore](../../../../modules/file-system/parallelstore/) module for similar functionality. To help with this transition, see the Parallelstore example blueprints ([pfs-parallelstore.yaml](../../../../examples/pfs-parallelstore.yaml) and [ps-slurm.yaml](../../../../examples/ps-slurm.yaml)). If the external [Google Cloud DAOS](https://github.com/daos-stack/google-cloud-daos/tree/main) repository is necessary, we recommend using the last Cluster Toolkit [v1.41.0](https://github.com/GoogleCloudPlatform/cluster-toolkit/releases/tag/v1.41.0). diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/README.md deleted file mode 100644 index 2e3fe7e933..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/README.md +++ /dev/null @@ -1,170 +0,0 @@ -## Description - -This module creates a [Google Cloud Storage (GCS) bucket](https://cloud.google.com/storage). - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). - -### Example - -The following example will create a bucket named `simulation-results-xxxxxxxx`, -where `xxxxxxxx` is a randomly generated id. - -```yaml - - id: bucket - source: community/modules/file-system/cloud-storage-bucket - settings: - name_prefix: simulation-results - random_suffix: true -``` - -> **_NOTE:_** Use of `random_suffix` may cause the following error when used -> with other modules: -> `value depends on resource attributes that cannot be determined until apply`. -> To resolve this set `random_suffix` to `false` (default). - - - -> **_NOTE:_** Bucket namespace is shared by all users of Google Cloud so it is -> possible to have a bucket name clash with an existing bucket that is not in -> your project. To resolve this try to use a more unique name, or set the -> `random_suffix` variable to `true`. - -## Naming of Bucket - -There are potentially three parts to the bucket name. Each of these parts are -configurable in the blueprint. - -1. A **custom prefix**, provided by the user in the blueprint \ -Provide the custom prefix using the `name_prefix` setting. - -1. The **deployment name**, included by default \ -The deployment name can be excluded by setting `use_deployment_name_in_bucket_name: false`. - -1. A **random id** suffix, excluded by default \ -The random id can be included by setting `random_suffix: true`. - -If none of these are provided (no `name_prefix`, -`use_deployment_name_in_bucket_name: false`, & `random_suffix: false`), then the -bucket name will default to `no-bucket-name-provided`. - -Since bucket namespace is shared by all users of Google Cloud, it is more likely -to experience naming clashes than with other resources. In many cases, adding -the `random_suffix` will resolve the naming clash issue. - -> **Warning**: If a bucket is created with a `random_suffix` and then used as -> the bucket for a startup script in the same deployment group this will cause a -> `not known at apply time` error in terraform. The solution is to either create -> the bucket in a separate deployment group or to remove the random suffix. - -## Mounting - -To mount the Cloud Storage bucket you must first ensure that the GCS Fuse client -has been installed and then call the proper `mount` command. - -Both of these steps are automatically handled with the use of the `use` command -in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in -the network storage doc for a complete list of supported modules. - -If mounting is not automatically handled as described above, the -`cloud-storage-bucket` module outputs runners that can be used with the -`startup-script` module to install the client and mount the file system. See the -following example: - -```yaml - - id: bucket - source: community/modules/file-system/cloud-storage-bucket - settings: {local_mount: /data} - - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: - - $(bucket.client_install_runner) - - $(bucket.mount_runner) -``` - -[matrix]: ../../../../docs/network_storage.md#compatibility-matrix - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [google](#requirement\_google) | >= 3.83 | -| [google-beta](#requirement\_google-beta) | >= 6.9.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [google-beta](#provider\_google-beta) | >= 6.9.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_storage_bucket) | resource | -| [google_storage_bucket_iam_binding.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_binding) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [autoclass](#input\_autoclass) | Configure bucket autoclass setup

The autoclass config supports automatic transitions of objects in the bucket to appropriate storage classes based on each object's access pattern.

The terminal storage class defines that objects in the bucket eventually transition to if they are not read for a certain length of time.
Supported values include: 'NEARLINE', 'ARCHIVE' (Default 'NEARLINE')

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/autoclass | 
object({
    enabled                = optional(bool, false)
    terminal_storage_class = optional(string, null)
  })
| 
{
  "enabled": false
}
| no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment; used as part of name of the GCS bucket. | `string` | n/a | yes | -| [enable\_hierarchical\_namespace](#input\_enable\_hierarchical\_namespace) | If true, enables hierarchical namespace for the bucket. This option must be configured during the initial creation of the bucket. | `bool` | `false` | no | -| [enable\_object\_retention](#input\_enable\_object\_retention) | If true, enables retention policy at per object level for the bucket.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/object-lock | `bool` | `false` | no | -| [enable\_versioning](#input\_enable\_versioning) | If true, enables versioning for the bucket. | `bool` | `false` | no | -| [force\_destroy](#input\_force\_destroy) | If true will destroy bucket with all objects stored within. | `bool` | `false` | no | -| [labels](#input\_labels) | Labels to add to the GCS bucket. Key-value pairs. | `map(string)` | n/a | yes | -| [lifecycle\_rules](#input\_lifecycle\_rules) | List of config to manage data lifecycle rules for the bucket. For more details: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket.html#nested_lifecycle_rule | 
list(object({
    # Object with keys:
    # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass.
    # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule.
    action = object({
      type          = string
      storage_class = optional(string)
    })

    # Object with keys:
    # - age - (Optional) Minimum age of an object in days to satisfy this condition.
    # - send_age_if_zero - (Optional) While set true, num_newer_versions value will be sent in the request even for zero value of the field.
    # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition.
    # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY".
    # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE, STANDARD, DURABLE_REDUCED_AVAILABILITY.
    # - matches_prefix - (Optional) One or more matching name prefixes to satisfy this condition.
    # - matches_suffix - (Optional) One or more matching name suffixes to satisfy this condition.
    # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition.
    # - custom_time_before - (Optional) A date in the RFC 3339 format YYYY-MM-DD. This condition is satisfied when the customTime metadata for the object is set to an earlier date than the date used in this lifecycle condition.
    # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true.
    # - days_since_noncurrent_time - (Optional) Relevant only for versioned objects. Number of days elapsed since the noncurrent timestamp of an object.
    # - noncurrent_time_before - (Optional) Relevant only for versioned objects. The date in RFC 3339 (e.g. 2017-06-13) when the object became nonconcurrent.
    condition = object({
      age                        = optional(number)
      send_age_if_zero           = optional(bool)
      created_before             = optional(string)
      with_state                 = optional(string)
      matches_storage_class      = optional(string)
      matches_prefix             = optional(string)
      matches_suffix             = optional(string)
      num_newer_versions         = optional(number)
      custom_time_before         = optional(string)
      days_since_custom_time     = optional(number)
      days_since_noncurrent_time = optional(number)
      noncurrent_time_before     = optional(string)
    })
  }))
| `[]` | no | -| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | -| [mount\_options](#input\_mount\_options) | Mount options to be put in fstab. Note: `implicit_dirs` makes it easier to work with objects added by other tools, but there is a performance impact. See: [more information](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#implicit-directories) | `string` | `"defaults,_netdev,implicit_dirs"` | no | -| [name\_prefix](#input\_name\_prefix) | Name Prefix. | `string` | `null` | no | -| [project\_id](#input\_project\_id) | ID of project in which GCS bucket will be created. | `string` | n/a | yes | -| [public\_access\_prevention](#input\_public\_access\_prevention) | Bucket public access can be controlled by setting a value of either `inherited` or `enforced`.
When set to `enforced`, public access to the bucket is blocked.
If set to `inherited`, the bucket's public access prevention depends on whether it is subject to the organization policy constraint for public access prevention.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/public-access-prevention | `string` | `null` | no | -| [random\_suffix](#input\_random\_suffix) | If true, a random id will be appended to the suffix of the bucket name. | `bool` | `false` | no | -| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | -| [retention\_policy\_period](#input\_retention\_policy\_period) | If defined, this will configure retention\_policy with retention\_period for the bucket, value must be in between 1 and 3155760000(100 years) seconds.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/bucket-lock | `number` | `null` | no | -| [soft\_delete\_retention\_duration](#input\_soft\_delete\_retention\_duration) | If defined, this will configure soft\_delete\_policy with retention\_duration\_seconds for the bucket, value can be 0 or in between 604800(7 days) and 7776000(90 days).
Setting a 0 duration disables soft delete, meaning any deleted objects will be permanently deleted.

See Cloud documentation for more details:

https://cloud.google.com/storage/docs/soft-delete | `number` | `null` | no | -| [storage\_class](#input\_storage\_class) | The storage class of the GCS bucket. | `string` | `"REGIONAL"` | no | -| [uniform\_bucket\_level\_access](#input\_uniform\_bucket\_level\_access) | Allow uniform control access to the bucket. | `bool` | `true` | no | -| [use\_deployment\_name\_in\_bucket\_name](#input\_use\_deployment\_name\_in\_bucket\_name) | If true, the deployment name will be included as part of the bucket name. This helps prevent naming clashes across multiple deployments. | `bool` | `true` | no | -| [viewers](#input\_viewers) | A list of additional accounts that can read packages from this bucket | `set(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [client\_install\_runner](#output\_client\_install\_runner) | Runner that performs client installation needed to use gcs fuse. | -| [gcs\_bucket\_name](#output\_gcs\_bucket\_name) | Bucket name. | -| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | The gsutil bucket path with format of `gs://`. | -| [mount\_runner](#output\_mount\_runner) | Runner that mounts the cloud storage bucket with gcs fuse. | -| [network\_storage](#output\_network\_storage) | Describes a remote network storage to be mounted by fs-tab. | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/main.tf deleted file mode 100644 index 81ba0ca6a9..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/main.tf +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "cloud-storage-bucket", ghpc_role = "file-system" }) -} - -locals { - prefix = var.name_prefix != null ? var.name_prefix : "" - deployment = var.use_deployment_name_in_bucket_name ? var.deployment_name : "" - suffix = var.random_suffix ? random_id.resource_name_suffix.hex : "" - first_dash = (local.prefix != "" && (local.deployment != "" || local.suffix != "")) ? "-" : "" - second_dash = local.deployment != "" && local.suffix != "" ? "-" : "" - composite_name = "${local.prefix}${local.first_dash}${local.deployment}${local.second_dash}${local.suffix}" - name = local.composite_name == "" ? "no-bucket-name-provided" : local.composite_name -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_storage_bucket" "bucket" { - provider = google-beta - project = var.project_id - name = local.name - uniform_bucket_level_access = var.uniform_bucket_level_access - location = var.region - storage_class = var.storage_class - labels = local.labels - force_destroy = var.force_destroy - public_access_prevention = var.public_access_prevention - enable_object_retention = var.enable_object_retention - hierarchical_namespace { - enabled = var.enable_hierarchical_namespace - } - - dynamic "autoclass" { - for_each = var.autoclass.enabled ? [1] : [] - content { - enabled = var.autoclass.enabled - terminal_storage_class = var.autoclass.terminal_storage_class - } - } - - dynamic "soft_delete_policy" { - for_each = var.soft_delete_retention_duration == null ? [] : [1] - content { - retention_duration_seconds = var.soft_delete_retention_duration - } - } - - dynamic "retention_policy" { - for_each = var.retention_policy_period == null ? [] : [1] - content { - retention_period = var.retention_policy_period - } - } - - dynamic "versioning" { - for_each = var.enable_versioning ? [1] : [] - content { - enabled = var.enable_versioning - } - } - - dynamic "lifecycle_rule" { - for_each = var.lifecycle_rules - content { - action { - type = lifecycle_rule.value.action.type - storage_class = lookup(lifecycle_rule.value.action, "storage_class", null) - } - condition { - age = lookup(lifecycle_rule.value.condition, "age", null) - send_age_if_zero = lookup(lifecycle_rule.value.condition, "send_age_if_zero", null) - created_before = lookup(lifecycle_rule.value.condition, "created_before", null) - with_state = lookup(lifecycle_rule.value.condition, "with_state", contains(keys(lifecycle_rule.value.condition), "is_live") ? (lifecycle_rule.value.condition["is_live"] ? "LIVE" : null) : null) - matches_storage_class = lifecycle_rule.value.condition["matches_storage_class"] != null ? split(",", lifecycle_rule.value.condition["matches_storage_class"]) : null - matches_prefix = lifecycle_rule.value.condition["matches_prefix"] != null ? split(",", lifecycle_rule.value.condition["matches_prefix"]) : null - matches_suffix = lifecycle_rule.value.condition["matches_suffix"] != null ? split(",", lifecycle_rule.value.condition["matches_suffix"]) : null - num_newer_versions = lookup(lifecycle_rule.value.condition, "num_newer_versions", null) - custom_time_before = lookup(lifecycle_rule.value.condition, "custom_time_before", null) - days_since_custom_time = lookup(lifecycle_rule.value.condition, "days_since_custom_time", null) - days_since_noncurrent_time = lookup(lifecycle_rule.value.condition, "days_since_noncurrent_time", null) - noncurrent_time_before = lookup(lifecycle_rule.value.condition, "noncurrent_time_before", null) - } - } - } - - lifecycle { - precondition { - condition = !var.autoclass.enabled || !var.enable_hierarchical_namespace - error_message = "Hierarchical namespace is not compatible with Autoclass enabled." - } - - precondition { - condition = !var.enable_hierarchical_namespace || var.uniform_bucket_level_access - error_message = "Hierarchical namespace is not compatible with Uniform bucket level access disabled." - } - - precondition { - condition = !var.enable_versioning || !var.enable_hierarchical_namespace - error_message = "Hierarchical namespace is not compatible with Object versioning enabled." - } - } -} - -resource "google_storage_bucket_iam_binding" "viewers" { - bucket = google_storage_bucket.bucket.name - role = "roles/storage.objectViewer" - members = var.viewers -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/outputs.tf deleted file mode 100644 index 29ddfef2d2..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/outputs.tf +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "network_storage" { - description = "Describes a remote network storage to be mounted by fs-tab." - value = { - remote_mount = local.name - local_mount = var.local_mount - fs_type = "gcsfuse" - mount_options = var.mount_options - server_ip = "" - client_install_runner = local.client_install_runner - mount_runner = local.mount_runner - } -} - -locals { - client_install_runner = { - "type" = "shell" - "content" = file("${path.module}/scripts/install-gcs-fuse.sh") - "destination" = "install-gcsfuse${replace(var.local_mount, "/", "_")}.sh" - } - - mount_runner = { - "type" = "shell" - "destination" = "mount_gcs${replace(var.local_mount, "/", "_")}.sh" - "args" = "\"not-used\" \"${local.name}\" \"${var.local_mount}\" \"gcsfuse\" \"${var.mount_options}\"" - "content" = file("${path.module}/scripts/mount.sh") - } -} - -output "client_install_runner" { - description = "Runner that performs client installation needed to use gcs fuse." - value = local.client_install_runner -} - -output "mount_runner" { - description = "Runner that mounts the cloud storage bucket with gcs fuse." - value = local.mount_runner -} - -output "gcs_bucket_path" { - description = "The gsutil bucket path with format of `gs://`." - # cannot use resource attribute, will cause lookup failure in startup-script - value = "gs://${local.name}" - - # needed to make sure bucket contents are deleted before bucket - depends_on = [ - google_storage_bucket.bucket - ] -} - -output "gcs_bucket_name" { - description = "Bucket name." - value = google_storage_bucket.bucket.name -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh deleted file mode 100644 index f8a990260b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/scripts/install-gcs-fuse.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -if [ ! "$(which gcsfuse)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ]; then - tee /etc/yum.repos.d/gcsfuse.repo >/dev/null </dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/variables.tf deleted file mode 100644 index 9804e4b268..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/variables.tf +++ /dev/null @@ -1,254 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which GCS bucket will be created." - type = string -} - -variable "deployment_name" { - description = "Name of the HPC deployment; used as part of name of the GCS bucket." - type = string -} - -variable "region" { - description = "The region to deploy to" - type = string -} - -variable "labels" { - description = "Labels to add to the GCS bucket. Key-value pairs." - type = map(string) -} - -variable "local_mount" { - description = "The mount point where the contents of the device may be accessed after mounting." - type = string - default = "/mnt" -} - -variable "mount_options" { - description = "Mount options to be put in fstab. Note: `implicit_dirs` makes it easier to work with objects added by other tools, but there is a performance impact. See: [more information](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#implicit-directories)" - type = string - default = "defaults,_netdev,implicit_dirs" -} - -variable "name_prefix" { - description = "Name Prefix." - type = string - default = null -} - -variable "use_deployment_name_in_bucket_name" { - description = "If true, the deployment name will be included as part of the bucket name. This helps prevent naming clashes across multiple deployments." - type = bool - default = true -} - -variable "random_suffix" { - description = "If true, a random id will be appended to the suffix of the bucket name." - type = bool - default = false -} - -variable "force_destroy" { - description = "If true will destroy bucket with all objects stored within." - type = bool - default = false -} - -variable "viewers" { - description = "A list of additional accounts that can read packages from this bucket" - type = set(string) - default = [] - - validation { - error_message = "All bucket viewers must be in IAM style: user:user@example.com, serviceAccount:sa@example.com, or group:group@example.com." - condition = alltrue([ - for viewer in var.viewers : length(regexall("^(user|serviceAccount|group):", viewer)) > 0 - ]) - } -} - -variable "enable_hierarchical_namespace" { - description = "If true, enables hierarchical namespace for the bucket. This option must be configured during the initial creation of the bucket." - type = bool - default = false -} - -variable "uniform_bucket_level_access" { - description = "Allow uniform control access to the bucket." - type = bool - default = true -} - -variable "storage_class" { - description = "The storage class of the GCS bucket." - type = string - default = "REGIONAL" - validation { - condition = contains([ - "STANDARD", - "MULTI_REGIONAL", - "REGIONAL", - "NEARLINE", - "COLDLINE", - "ARCHIVE" - ], var.storage_class) - error_message = "Allowed values for GCS storage_class are 'STANDARD', 'MULTI_REGIONAL', 'REGIONAL', 'NEARLINE', 'COLDLINE', 'ARCHIVE'.\nhttps://cloud.google.com/storage/docs/storage-classes" - } -} - -variable "autoclass" { - description = <<-EOT - Configure bucket autoclass setup - - The autoclass config supports automatic transitions of objects in the bucket to appropriate storage classes based on each object's access pattern. - - The terminal storage class defines that objects in the bucket eventually transition to if they are not read for a certain length of time. - Supported values include: 'NEARLINE', 'ARCHIVE' (Default 'NEARLINE') - - See Cloud documentation for more details: - - https://cloud.google.com/storage/docs/autoclass - EOT - type = object({ - enabled = optional(bool, false) - terminal_storage_class = optional(string, null) - }) - default = { - enabled = false - } - nullable = false - validation { - condition = !can(coalesce(var.autoclass.terminal_storage_class)) || var.autoclass.enabled - error_message = "Cannot set bucket var.autoclass.terminal_storage_class unless var.autoclass.enabled is true" - } -} - -variable "public_access_prevention" { - description = <<-EOT - Bucket public access can be controlled by setting a value of either `inherited` or `enforced`. - When set to `enforced`, public access to the bucket is blocked. - If set to `inherited`, the bucket's public access prevention depends on whether it is subject to the organization policy constraint for public access prevention. - - See Cloud documentation for more details: - - https://cloud.google.com/storage/docs/public-access-prevention - EOT - type = string - default = null - validation { - condition = var.public_access_prevention == null ? true : contains([ - "inherited", - "enforced" - ], var.public_access_prevention) - error_message = "Allowed values for public_access_prevention are 'inherited', 'enforced'.\n" - } -} - -variable "soft_delete_retention_duration" { - description = <<-EOT - If defined, this will configure soft_delete_policy with retention_duration_seconds for the bucket, value can be 0 or in between 604800(7 days) and 7776000(90 days). - Setting a 0 duration disables soft delete, meaning any deleted objects will be permanently deleted. - - See Cloud documentation for more details: - - https://cloud.google.com/storage/docs/soft-delete - EOT - type = number - default = null - validation { - condition = var.soft_delete_retention_duration == null ? true : var.soft_delete_retention_duration == 0 || var.soft_delete_retention_duration >= 604800 && var.soft_delete_retention_duration <= 7776000 - error_message = "var.soft_delete_retention_duration value can be 0 or in between 604800(7 days) and 7776000(90 days)." - } -} - -variable "enable_versioning" { - description = "If true, enables versioning for the bucket." - type = bool - default = false -} - -variable "lifecycle_rules" { - description = "List of config to manage data lifecycle rules for the bucket. For more details: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket.html#nested_lifecycle_rule" - type = list(object({ - # Object with keys: - # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass. - # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule. - action = object({ - type = string - storage_class = optional(string) - }) - - # Object with keys: - # - age - (Optional) Minimum age of an object in days to satisfy this condition. - # - send_age_if_zero - (Optional) While set true, num_newer_versions value will be sent in the request even for zero value of the field. - # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition. - # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY". - # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE, STANDARD, DURABLE_REDUCED_AVAILABILITY. - # - matches_prefix - (Optional) One or more matching name prefixes to satisfy this condition. - # - matches_suffix - (Optional) One or more matching name suffixes to satisfy this condition. - # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition. - # - custom_time_before - (Optional) A date in the RFC 3339 format YYYY-MM-DD. This condition is satisfied when the customTime metadata for the object is set to an earlier date than the date used in this lifecycle condition. - # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true. - # - days_since_noncurrent_time - (Optional) Relevant only for versioned objects. Number of days elapsed since the noncurrent timestamp of an object. - # - noncurrent_time_before - (Optional) Relevant only for versioned objects. The date in RFC 3339 (e.g. 2017-06-13) when the object became nonconcurrent. - condition = object({ - age = optional(number) - send_age_if_zero = optional(bool) - created_before = optional(string) - with_state = optional(string) - matches_storage_class = optional(string) - matches_prefix = optional(string) - matches_suffix = optional(string) - num_newer_versions = optional(number) - custom_time_before = optional(string) - days_since_custom_time = optional(number) - days_since_noncurrent_time = optional(number) - noncurrent_time_before = optional(string) - }) - })) - default = [] -} - -variable "retention_policy_period" { - description = <<-EOT - If defined, this will configure retention_policy with retention_period for the bucket, value must be in between 1 and 3155760000(100 years) seconds. - - See Cloud documentation for more details: - - https://cloud.google.com/storage/docs/bucket-lock - EOT - type = number - default = null - validation { - condition = var.retention_policy_period == null ? true : var.retention_policy_period > 0 && var.retention_policy_period <= 3155760000 - error_message = "var.soft_delete_policy_retention_duration value must be in between 1 and 3155760000(100 years) seconds." - } -} - -variable "enable_object_retention" { - description = <<-EOT - If true, enables retention policy at per object level for the bucket. - - See Cloud documentation for more details: - - https://cloud.google.com/storage/docs/object-lock - EOT - type = bool - default = false -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/versions.tf deleted file mode 100644 index e80c28ca47..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/cloud-storage-bucket/versions.tf +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - google-beta = { - source = "hashicorp/google-beta" - version = ">= 6.9.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.57.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.57.0" - } - required_version = ">= 0.14.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/README.md deleted file mode 100644 index 66aaaa46af..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/README.md +++ /dev/null @@ -1,152 +0,0 @@ -## Description - -This module creates a Network File Sharing (NFS) file system based on a VM -instance and [compute disk][disk]. This file system can share directories and -files with other clients over a network. `nfs-server` can be used by -[vm-instance](../../../../modules/compute/vm-instance/README.md) and SchedMD -community modules that create compute VMs. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../../docs/network_storage.md). - -If you are using Hyperdisk storage, check the possible disk size, IOPS, and throughput values for each disk type in the [Hyperdisk limits documentation](https://cloud.google.com/compute/docs/disks/hyperdisks#limits-disk). - -> **_WARNING:_** This module has only been tested against the HPC centos7 OS -> disk image (the default). Using other images may work, but have not been -> verified. - -[disk]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk - -### Example - -```yaml -- id: homefs - source: community/modules/file-system/nfs-server - use: [network1] -``` - -This creates a NFS on a virtual machine which allow other VMs to mount the -volume as an external file system. - -> **_NOTE:_** All disks are destroyed along with the instance, during a `gcluster destroy`/`terraform destroy` event. However, you can setup data retention with `create_boot_snapshot_before_destroy` (boot disk) and `create_snapshot_before_destroy` (data disk). - -## Mounting - -To mount the NFS Server you must first ensure that the NFS client has been -installed the and then call the proper `mount` command. - -Both of these steps are automatically handled with the use of the `use` command -in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in -the network storage doc for a complete list of supported modules. -See the [hpc-centos-ss.yaml] test config for an example of using this module -with a `vm-instance` module. - -If mounting is not automatically handled as described above, the `nfs-server` -module outputs runners that can be used with the startup-script module to -install the client and mount the file system. See the following example: - -```yaml - - id: nfs - source: community/modules/file-system/nfs-server - use: [network1] - settings: {local_mounts: [/mnt1]} - - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: - - $(nfs.install_nfs_client_runner) - - $(nfs.mount_runner) - -``` - -[hpc-centos-ss.yaml]: ../../../../tools/validate_configs/test_configs/hpc-centos-ss.yaml -[matrix]: ../../../../docs/network_storage.md#compatibility-matrix - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [google](#requirement\_google) | >= 6.14 | -| [null](#requirement\_null) | >= 3.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.14 | -| [null](#provider\_null) | >= 3.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_disk.attached_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [google_compute_disk.boot_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [google_compute_instance.compute_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance) | resource | -| [null_resource.image](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [google_compute_default_service_account.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_default_service_account) | data source | -| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [auto\_delete\_disk](#input\_auto\_delete\_disk) | DEPRECATED: Whether or not the NFS disk should be auto-deleted | `string` | `null` | no | -| [boot\_disk\_size](#input\_boot\_disk\_size) | Storage size in GB for the boot disk | `number` | `null` | no | -| [boot\_disk\_type](#input\_boot\_disk\_type) | Storage type for the boot disk | `string` | `null` | no | -| [create\_boot\_snapshot\_before\_destroy](#input\_create\_boot\_snapshot\_before\_destroy) | Whether to create a snapshot before destroying the boot disk | `bool` | `false` | no | -| [create\_snapshot\_before\_destroy](#input\_create\_snapshot\_before\_destroy) | Whether to create a snapshot before destroying the NFS data disk | `bool` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the NFS instance if no name is specified. | `string` | n/a | yes | -| [disk\_size](#input\_disk\_size) | Storage size in GB for the NFS data disk | `number` | `"100"` | no | -| [image](#input\_image) | DEPRECATED: The VM image used by the NFS server | `string` | `null` | no | -| [instance\_image](#input\_instance\_image) | The VM image used by the NFS server.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | -| [labels](#input\_labels) | Labels to add to the NFS instance. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_mounts](#input\_local\_mounts) | Mountpoint for this NFS compute instance | `list(string)` | 
[
  "/data"
]
| no | -| [machine\_type](#input\_machine\_type) | Type of the VM instance to use | `string` | `"n2d-standard-2"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | -| [name](#input\_name) | The resource name of the instance. | `string` | `null` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the NFS VM. | `string` | `"default"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [provisioned\_iops](#input\_provisioned\_iops) | Provisioned IOPS for the NFS data disk if using Extreme PD or Hyperdisk Balanced/ML/Throughput | `number` | `null` | no | -| [provisioned\_throughput](#input\_provisioned\_throughput) | Provisioned throughput for the NFS data disk if using Hyperdisk Balanced/Extreme | `number` | `null` | no | -| [scopes](#input\_scopes) | Scopes to apply to the controller | `list(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [service\_account](#input\_service\_account) | Service Account for the NFS server | `string` | `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the NFS VM. | `string` | `null` | no | -| [type](#input\_type) | Storage type for the NFS data disk | `string` | `"pd-ssd"` | no | -| [zone](#input\_zone) | The zone name where the NFS instance located in. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | -| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | -| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | -| [network\_storage](#output\_network\_storage) | export of all desired folder directories | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/main.tf deleted file mode 100644 index a00d2681ba..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/main.tf +++ /dev/null @@ -1,131 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "nfs-server", ghpc_role = "file-system" }) -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -locals { - name = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" - server_ip = google_compute_instance.compute_instance.network_interface[0].network_ip - fs_type = "nfs" - mount_options = "defaults,hard,intr" - install_nfs_client_runners = [for mount in var.local_mounts : - { - "type" = "shell" - "source" = "${path.module}/scripts/install-nfs-client.sh" - "destination" = "install-nfs${replace(mount, "/", "_")}.sh" - } - ] - mount_runners = [for mount in var.local_mounts : - { - "type" = "shell" - "source" = "${path.module}/scripts/mount.sh" - "args" = "\"${local.server_ip}\" \"/exports${mount}\" \"${mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" - "destination" = "mount${replace(mount, "/", "_")}.sh" - } - ] - ansible_mount_runner = { - "type" = "ansible-local" - "source" = "${path.module}/scripts/mount.yaml" - "destination" = "mount.yaml" - } -} - -data "google_compute_default_service_account" "default" {} - -resource "google_compute_disk" "attached_disk" { - project = var.project_id - name = "${local.name}-nfs-instance-disk" - size = var.disk_size - type = var.type - zone = var.zone - labels = local.labels - provisioned_iops = var.provisioned_iops - provisioned_throughput = var.provisioned_throughput - create_snapshot_before_destroy = var.create_snapshot_before_destroy -} - -data "google_compute_image" "compute_image" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = var.instance_image.project -} - -resource "null_resource" "image" { - triggers = { - name = try(var.instance_image.name, null), - family = try(var.instance_image.family, null), - project = var.instance_image.project - } -} - -resource "google_compute_disk" "boot_disk" { - project = var.project_id - - name = "${local.name}-boot-disk" - size = var.boot_disk_size - type = var.boot_disk_type - image = data.google_compute_image.compute_image.self_link - labels = local.labels - zone = var.zone - create_snapshot_before_destroy = var.create_boot_snapshot_before_destroy - - lifecycle { - replace_triggered_by = [null_resource.image] - ignore_changes = [ - image - ] - } -} - -resource "google_compute_instance" "compute_instance" { - project = var.project_id - name = "${local.name}-nfs-instance" - zone = var.zone - machine_type = var.machine_type - - boot_disk { - auto_delete = false - source = google_compute_disk.boot_disk.self_link - device_name = google_compute_disk.boot_disk.name - } - - attached_disk { - source = google_compute_disk.attached_disk.id - device_name = "attached_disk" - } - - network_interface { - network = var.network_self_link - subnetwork = var.subnetwork_self_link - } - - service_account { - email = var.service_account == null ? data.google_compute_default_service_account.default.email : var.service_account - scopes = var.scopes - } - - metadata = var.metadata - metadata_startup_script = templatefile("${path.module}/scripts/install-nfs-server.sh.tpl", { local_mounts = var.local_mounts }) - - labels = local.labels -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/outputs.tf deleted file mode 100644 index e23b94e2b2..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/outputs.tf +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ -# render the content for each folder -output "network_storage" { - description = "export of all desired folder directories" - value = [for i, mount in var.local_mounts : { - remote_mount = "/exports${mount}" - local_mount = mount - fs_type = local.fs_type - mount_options = local.mount_options - server_ip = local.server_ip - client_install_runner = local.install_nfs_client_runners[i] - mount_runner = local.mount_runners[i] - } - ] -} - -output "install_nfs_client" { - description = "Script for installing NFS client" - value = file("${path.module}/scripts/install-nfs-client.sh") -} - -output "install_nfs_client_runner" { - description = "Runner to install NFS client using the startup-script module" - value = local.install_nfs_client_runners[0] -} - -output "mount_runner" { - description = <<-EOT - Runner to mount the file-system using an ansible playbook. The startup-script - module will automatically handle installation of ansible. - - id: example-startup-script - source: modules/scripts/startup-script - settings: - runners: - - $(your-fs-id.mount_runner) - ... - EOT - value = local.ansible_mount_runner -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh deleted file mode 100644 index 9f842c5d7c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-client.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ! "$(which mount.nfs)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || - [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - major_version=$(rpm -E "%{rhel}") - enable_repo="" - if [ "${major_version}" -eq "7" ]; then - enable_repo="base,epel" - elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then - enable_repo="baseos" - else - echo "Unsupported version of centos/RHEL/Rocky" - return 1 - fi - yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils - elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get -y install nfs-common - else - echo 'Unsuported distribution' - return 1 - fi -fi diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl deleted file mode 100644 index 1b06a5f032..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/install-nfs-server.sh.tpl +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -ex - -if [ ! -d "/exports" ]; then # first load, format and mount the disk - # See https://cloud.google.com/compute/docs/disks/add-persistent-disk - uuid=$(uuidgen) - mkfs.ext4 -F -m 0 -U "$uuid" -E lazy_itable_init=0,lazy_journal_init=0,discard /dev/disk/by-id/google-attached_disk - - mkdir /exports - echo "UUID=$uuid /exports ext4 discard,defaults 0 0" >> /etc/fstab - mount --target /exports/ - - %{ for mount in local_mounts ~} - mkdir -p /exports${mount} - chmod 755 /exports${mount} - echo '/exports${mount} *(rw,sync,no_root_squash)' >> "/etc/exports" - %{ endfor ~} -fi - -systemctl start nfs-server rpcbind -systemctl enable nfs-server -exportfs -r diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh deleted file mode 100644 index e2509fb4a1..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -SERVER_IP=$1 -REMOTE_MOUNT=$2 -LOCAL_MOUNT=$3 -FS_TYPE=$4 -MOUNT_OPTIONS=$5 - -[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" - -if [ "${FS_TYPE}" = "gcsfuse" ]; then - FS_SPEC="${REMOTE_MOUNT}" -else - FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" -fi - -SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" -EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" - -grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false -grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false -findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml deleted file mode 100644 index f7fbe58d5e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/scripts/mount.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Mounts the file systems specified in the metadata network_storage key - hosts: localhost - become: true - vars: - meta_key: "network_storage" - url: "http://metadata.google.internal/computeMetadata/v1/instance/attributes" - tasks: - - name: Read metadata network_storage information - ansible.builtin.uri: - url: "{{ url }}/{{ meta_key }}" - method: GET - headers: - Metadata-Flavor: "Google" - register: storage - - name: Mount file systems - ansible.posix.mount: - src: "{{ item.server_ip }}:/{{ item.remote_mount }}" - path: "{{ item.local_mount }}" - opts: "{{ item.mount_options }}" - boot: true - fstype: "{{ item.fs_type }}" - state: "mounted" - loop: "{{ storage.json }}" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/variables.tf deleted file mode 100644 index 9a58da641e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/variables.tf +++ /dev/null @@ -1,194 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "deployment_name" { - description = "Name of the HPC deployment, used as name of the NFS instance if no name is specified." - type = string -} - -variable "name" { - description = "The resource name of the instance." - type = string - default = null -} - -variable "zone" { - description = "The zone name where the NFS instance located in." - type = string -} - -variable "boot_disk_size" { - description = "Storage size in GB for the boot disk" - type = number - default = null -} - -variable "boot_disk_type" { - description = "Storage type for the boot disk" - type = string - default = null -} - -variable "create_boot_snapshot_before_destroy" { - description = "Whether to create a snapshot before destroying the boot disk" - type = bool - default = false -} - -variable "disk_size" { - description = "Storage size in GB for the NFS data disk" - type = number - default = "100" -} - -variable "type" { - description = "Storage type for the NFS data disk" - type = string - default = "pd-ssd" -} - -variable "create_snapshot_before_destroy" { - description = "Whether to create a snapshot before destroying the NFS data disk" - type = bool - default = false -} - -variable "provisioned_iops" { - description = "Provisioned IOPS for the NFS data disk if using Extreme PD or Hyperdisk Balanced/ML/Throughput" - type = number - default = null -} - -variable "provisioned_throughput" { - description = "Provisioned throughput for the NFS data disk if using Hyperdisk Balanced/Extreme" - type = number - default = null -} - -# Deprecated, replaced by instance_image -# tflint-ignore: terraform_unused_declarations -variable "image" { - description = "DEPRECATED: The VM image used by the NFS server" - type = string - default = null - - validation { - condition = var.image == null - error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." - } -} - -variable "instance_image" { - description = <<-EOD - The VM image used by the NFS server. - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "cloud-hpc-image-public" - family = "hpc-rocky-linux-8" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -# Deprecated, replaced by create_snapshot_before_destroy and create_boot_snapshot_before_destroy -# tflint-ignore: terraform_unused_declarations -variable "auto_delete_disk" { - description = "DEPRECATED: Whether or not the NFS disk should be auto-deleted" - type = string - default = null - - validation { - condition = var.auto_delete_disk == null - error_message = "The 'var.auto_delete_disk' setting is broken in Cluster Toolkit versions >1.25.0 and deprecated in versions >1.48.0, please use 'var.create_snapshot_before_destroy' and 'var.create_boot_snapshot_before_destroy' instead." - } -} - -variable "network_self_link" { - description = "The self link of the network to attach the NFS VM." - type = string - default = "default" -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork to attach the NFS VM." - type = string - default = null -} - -variable "machine_type" { - description = "Type of the VM instance to use" - type = string - default = "n2d-standard-2" -} - -variable "labels" { - description = "Labels to add to the NFS instance. Key-value pairs." - type = map(string) -} - -variable "metadata" { - description = "Metadata, provided as a map" - type = map(string) - default = {} -} - -variable "service_account" { - description = "Service Account for the NFS server" - type = string - default = null -} - -variable "scopes" { - description = "Scopes to apply to the controller" - type = list(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -variable "local_mounts" { - description = "Mountpoint for this NFS compute instance" - type = list(string) - default = ["/data"] - - validation { - condition = alltrue([ - for m in var.local_mounts : substr(m, 0, 1) == "/" - ]) - error_message = "Local mountpoints have to start with '/'." - } - validation { - condition = length(var.local_mounts) > 0 - error_message = "At least one local mount must be specified in var.local_mounts." - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/versions.tf deleted file mode 100644 index 8c8c39fda7..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/file-system/nfs-server/versions.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.14" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:nfs-server/v1.57.0" - } - - required_version = ">= 0.14.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb deleted file mode 100644 index f13726f691..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/FSI_MonteCarlo.ipynb +++ /dev/null @@ -1,125 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "project_id = \"${project_id}\"\n", - "dataset_id = \"${dataset_id}\"\n", - "table_id = \"${table_id}\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ONI1Xo0-KtAD", - "outputId": "fb9ca475-e4ec-4cd0-e0e6-14f409eefd7a" - }, - "outputs": [], - "source": [ - "from google.cloud import bigquery\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "\n", - "client = bigquery.Client(project=project_id)\n", - "\n", - "df = client.query(f'''\n", - "SELECT ticker, cast(price AS FLOAT64) AS price, CAST(OFFSET as INTEGER) AS offset, start_date, end_date, iteration\n", - "FROM `{project_id}.{dataset_id}.{table_id}`,\n", - "UNNEST(simulation_results) as NUMERIC with OFFSET\n", - "WHERE epoch_time IN\n", - " # Get the latest simulation runs for each Ticker Symbol\n", - "(SELECT MAX(epoch_time) FROM `{project_id}.{dataset_id}.{table_id}` GROUP BY ticker)\n", - "'''\n", - ").to_dataframe()\n", - "# Display the data\n", - "df" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Define a function to plot the data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "def plot_ticker(t,df):\n", - "\n", - " dtf = df[(df.ticker==t) &(df.offset == 250)].price.describe(include=[np.float64], percentiles=[.05, .01, .001])\n", - " cellText = []\n", - " for v in dtf.values:\n", - " cellText.append([v])\n", - " \n", - " pltf = df[df.ticker==t].pivot(index='offset', columns='iteration', values='price')\n", - " \n", - " fig = plt.figure(figsize=(10,5))\n", - " ax1 = fig.add_subplot(122)\n", - " pltf.plot(legend=False, ax=ax1, xlabel='Time(days)', ylabel='US$', title=f\"{ df[(df.ticker == t) & (df.offset == 0) & (df.iteration == 4)]}\")\n", - " ax2 = fig.add_subplot(121)\n", - " font_size=10\n", - " bbox=[0, 0, .5, 1]\n", - " ax2.axis('off')\n", - " mpl_table = ax2.table(cellText = cellText, rowLabels=dtf.index.values, bbox=bbox)\n", - " mpl_table.auto_set_font_size(False)\n", - " mpl_table.set_fontsize(font_size)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 808 - }, - "id": "jvBmb_KceX7z", - "outputId": "42a3ba9f-b68f-4c7b-d928-0fedeed9216c" - }, - "outputs": [], - "source": [ - "ticker_list = df.ticker.unique()\n", - "for t in ticker_list:\n", - " plot_ticker(t,df)" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md deleted file mode 100644 index e54893a1bb..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/README.md +++ /dev/null @@ -1,97 +0,0 @@ -## Description - -Copy files to a target GCS bucket. - -Primarily used for FSI - MonteCarlo Tutorial **[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: -../docs/tutorials/fsi-montecarlo-on-batch/README.md - -## Usage -This copies the module files to the specified GCS bucket. It is expected that -the bucket will be mounted on the target VM. - -Some of the files are templates, and `main.tf` translates the files with the -passed variable values. This way the user does not have to change things like -pointing to the correct bigquery table or adding in the project_id. - -```yaml - - id: fsi_tutorial_files - source: community/modules/files/fsi-montecarlo-on-batch - use: [bq-dataset, bq-table, fsi_bucket, pubsub_topic] -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 3.83 | -| [http](#requirement\_http) | ~> 3.0 | -| [random](#requirement\_random) | ~> 3.0 | -| [template](#requirement\_template) | ~> 2.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [http](#provider\_http) | ~> 3.0 | -| [random](#provider\_random) | ~> 3.0 | -| [template](#provider\_template) | ~> 2.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.get_iteration_sh](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.get_mc_reqs](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.get_requirements](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.ipynb_obj_fsi](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.mc_obj_yaml](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.mc_run](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.run_batch_py](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [http_http.batch_py](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | -| [http_http.batch_requirements](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | -| [template_file.ipynb_fsi](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | -| [template_file.mc_run_py](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | -| [template_file.mc_run_yaml](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [dataset\_id](#input\_dataset\_id) | Bigquery dataset id | `string` | n/a | yes | -| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | Bucket name | `string` | `null` | no | -| [project\_id](#input\_project\_id) | ID of project in which GCS bucket will be created. | `string` | n/a | yes | -| [region](#input\_region) | Region to run project | `string` | n/a | yes | -| [table\_id](#input\_table\_id) | Bigquery table id | `string` | n/a | yes | -| [topic\_id](#input\_topic\_id) | Pubsub Topic Name | `string` | n/a | yes | -| [topic\_schema](#input\_topic\_schema) | Pubsub Topic schema | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh deleted file mode 100644 index 50aa865a31..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/iteration.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ticker=("GOOG" "AMZN" "MSFT" "NVDA" "META" "TSLA" "PEP" "COST") -echo "BI: $BATCH_TASK_INDEX" -echo "TI: ${ticker[$BATCH_TASK_INDEX]}" -python3 -m pip install -r /mnt/disks/fsi/mc_run_reqs.txt -python3 /mnt/disks/fsi/mc_run.py \ - --ticker "${ticker[$BATCH_TASK_INDEX]}" \ - --iterations 500 \ - --start_date 2022-01-01 diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf deleted file mode 100644 index 83dc7fe9cf..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/main.tf +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - bucket = replace(var.gcs_bucket_path, "gs://", "") -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -data "template_file" "mc_run_py" { - template = file("${path.module}/mc_run.tpl.py") - vars = { - project_id = var.project_id - topic_id = var.topic_id - topic_schema = var.topic_schema - dataset_id = var.dataset_id - table_id = var.table_id - } -} - -resource "google_storage_bucket_object" "mc_run" { - name = "mc_run.py" - content = data.template_file.mc_run_py.rendered - bucket = local.bucket -} - -data "template_file" "mc_run_yaml" { - template = file("${path.module}/mc_run.tpl.yaml") - vars = { - project_id = var.project_id - bucket_name = local.bucket - region = var.region - } -} - -resource "google_storage_bucket_object" "mc_obj_yaml" { - name = "mc_run.yaml" - content = data.template_file.mc_run_yaml.rendered - bucket = local.bucket -} - -data "template_file" "ipynb_fsi" { - template = file("${path.module}/FSI_MonteCarlo.ipynb") - vars = { - project_id = var.project_id - dataset_id = var.dataset_id - table_id = var.table_id - } -} -resource "google_storage_bucket_object" "ipynb_obj_fsi" { - name = "FSI_MonteCarlo.ipynb" - content = data.template_file.ipynb_fsi.rendered - bucket = local.bucket -} - -data "http" "batch_py" { - url = "https://raw.githubusercontent.com/GoogleCloudPlatform/scientific-computing-examples/main/python-batch/batch.py" -} - -resource "google_storage_bucket_object" "run_batch_py" { - name = "batch.py" - content = data.http.batch_py.response_body - bucket = local.bucket -} - -data "http" "batch_requirements" { - url = "https://raw.githubusercontent.com/GoogleCloudPlatform/scientific-computing-examples/main/python-batch/requirements.txt" -} - -resource "google_storage_bucket_object" "get_requirements" { - name = "requirements.txt" - content = data.http.batch_requirements.response_body - bucket = local.bucket -} - -resource "google_storage_bucket_object" "get_iteration_sh" { - name = "iteration.sh" - content = file("${path.module}/iteration.sh") - bucket = local.bucket -} - -resource "google_storage_bucket_object" "get_mc_reqs" { - name = "mc_run_reqs.txt" - content = file("${path.module}/mc_run_reqs.txt") - bucket = local.bucket -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py deleted file mode 100644 index 4e0a64e363..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Run MC simulation for VaR portfolio risk -""" - -import avro.schema -import io -import google.auth -import numpy -import time -import yfinance as yf - -from absl import app -from absl import flags -from avro.io import DatumWriter, BinaryEncoder, BinaryDecoder, DatumReader -from datetime import datetime -from datetime import timedelta -from google.cloud import pubsub_v1, bigquery -from google.cloud.pubsub import SchemaServiceClient - -PROJECT_ID = '${project_id}' -INCOMING_TOPIC_ID = '${topic_id}' -INCOMING_TOPIC_SCHEMA = '${topic_schema}' -DATASET_ID = '${dataset_id}' -TABLE_ID = '${table_id}' - - -FLAGS = flags.FLAGS - -flags.DEFINE_string("ticker", 'GOOG', "Nasdaq Stock Ticker to run, default GOOG") -flags.DEFINE_string("start_date", '2022-01-01' , "Start data for data query, default 2022-01-01") -flags.DEFINE_integer("calendar_days", 365 , "How many calendar days to include in the calculation") -flags.DEFINE_integer("epoch_time", f'{int(time.time())}' , "Epoch time, number of seconds since January 1st, 1970 at 00:00:00 UTC.") -flags.DEFINE_integer("iterations", 100 , "Number of iterations to run.") -flags.DEFINE_boolean("print_raw", False, "Dump raw data.") - -class VaRSimulator: - - def __init__(self): - pass - - def get_data(self): - self.get_historical_data_yahoo() - - def get_historical_data_yahoo(self): - - # get historical market data: https://pypi.org/project/yfinance/ - - self.raw_data = yf.Ticker(self.ticker).history(start=self.start_date, end=self.end_date ) - self.data = self.raw_data.Close - - def print_raw(self): - print(self.get_stats()) - print(type(self.raw_data)) - print(self.raw_data) - - def get_stats(self): - close = self.data - self.first = close[0] - self.last = close[-1] - self.trading_days = len(close) - self.cagr = (self.last / self.first) ** (365.0/self.calendar_days) -1.0 - self.volatility = self.data.pct_change().std() - return(self.first, self.last, self.trading_days, self.cagr, self.volatility) - - def run_simulation(self): - - returns = numpy.random.normal(self.cagr/self.trading_days, self.volatility, self.trading_days) + 1 - returns = numpy.insert(returns,0,1.0) - self.simulation_results = self.last * returns.cumprod() - return(self.simulation_results) - - def create_object(self): - self.object = { - "ticker": self.ticker, - "epoch_time": self.epoch_time, - "iteration": self.iteration, - "start_date": self.start_date, - "end_date": self.end_date, - "simulation_results": list(map(lambda x: {"price":x}, self.simulation_results)) - } - return(self.object) - - -class PubsubToBiquery: - - def __init__(self): - - the_time = int(time.time()) - - self.project_id = PROJECT_ID - - self.publisher_client = pubsub_v1.PublisherClient() - self.topic_path = self.publisher_client.topic_path(self.project_id, INCOMING_TOPIC_ID) - - self.schema_client = SchemaServiceClient() - self.schema_path = self.schema_client.schema_path(self.project_id, INCOMING_TOPIC_SCHEMA) - - pubsub_schema = self.schema_client.get_schema(request={"name": self.schema_path}) - avro_schema = avro.schema.parse(pubsub_schema.definition) - - self.writer = DatumWriter(avro_schema) - - - def publish_record(self,record): - - byte_stream = io.BytesIO() - encoder = BinaryEncoder(byte_stream) - self.writer.write(record, encoder) - data = byte_stream.getvalue() - byte_stream.flush() - future = self.publisher_client.publish(self.topic_path, data) - if(FLAGS.print_raw): - print(f"Published message ID: {future.result()}") - - -def main(argv): - - vr = VaRSimulator() - pbbq = PubsubToBiquery() - - vr.ticker =FLAGS.ticker - vr.start_date =FLAGS.start_date - vr.end_date =f'{(datetime.strptime(FLAGS.start_date,"%Y-%m-%d") + timedelta(days = FLAGS.calendar_days)).date()}' - vr.calendar_days = FLAGS.calendar_days - vr.epoch_time = FLAGS.epoch_time - vr.iteration = 1 - - vr.get_data() - vr.get_stats() - - for i in range(FLAGS.iterations): - vr.iteration = i - vr.run_simulation() - pbbq.publish_record(vr.create_object()) - - if(FLAGS.print_raw): - vr.print_raw() - - -if __name__ == "__main__": - """ This is executed when run from the command line """ - app.run(main) diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml deleted file mode 100644 index 7f7de4840b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run.tpl.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -project_id: "${project_id}" -region: "${region}" - -job_prefix: 'fsi-' -machine_type: "n2-standard-2" -volumes: -- {bucket_name: "${bucket_name}", gcs_path: "/mnt/disks/fsi"} - -container: - image_uri: "python" - entry_point: "/bin/bash" - commands: ["/mnt/disks/fsi/iteration.sh", "$BATCH_TASK_INDEX"] - -task_count: 8 #optional -parallelism: 4 #optional -task_count_per_node: 2 #optional -cpu_milli: 1000 #optional -memory_mib: 102400 #optional - - -labels: - env: "monte" - type: "carlo" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt deleted file mode 100644 index 105ed70ad2..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/mc_run_reqs.txt +++ /dev/null @@ -1,9 +0,0 @@ -absl-py -avro -google-auth -google-cloud -google-cloud-batch -google-cloud-pubsub -google-cloud-bigquery -yfinance -PyYAML diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml deleted file mode 100644 index 268c8faa9a..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [storage.googleapis.com] diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf deleted file mode 100644 index eddf3c9478..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/variables.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which GCS bucket will be created." - type = string -} - -variable "gcs_bucket_path" { - description = "Bucket name" - type = string - default = null -} - -variable "topic_id" { - description = "Pubsub Topic Name" - type = string -} - -variable "topic_schema" { - description = "Pubsub Topic schema" - type = string -} - -variable "dataset_id" { - description = "Bigquery dataset id" - type = string -} - -variable "table_id" { - description = "Bigquery table id" - type = string -} - -variable "region" { - description = "Region to run project" - type = string -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf deleted file mode 100644 index 07270f9cfe..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/files/fsi-montecarlo-on-batch/versions.tf +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - http = { - source = "hashicorp/http" - version = "~> 3.0" - } - template = { - source = "hashicorp/template" - version = "~> 2.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.57.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:fsi-montecarlo-on-batch/v1.57.0" - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md deleted file mode 100644 index ae8462d763..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/README.md +++ /dev/null @@ -1,100 +0,0 @@ -# Module: Slurm Instance - - - -- [Module: Slurm Instance](#module-slurm-instance) - - [Overview](#overview) - - [Module API](#module-api) - - - -## Overview - -This module creates a [compute instance](../../../../docs/glossary.md#vm) from -[instance template](../../../../docs/glossary.md#instance-template) for a -[Slurm cluster](../slurm_cluster/README.md). - -> **NOTE:** This module is only intended to be used by Slurm modules. For -> general usage, please consider using: -> -> - [terraform-google-modules/vm/google//modules/compute_instance](https://registry.terraform.io/modules/terraform-google-modules/vm/google/latest/submodules/compute_instance). -> **WARNING:** The source image is not modified. Make sure to use a compatible -> source image. - -## Module API - -For the terraform module API reference, please see -[README_TF.md](./README_TF.md). - - -Copyright (C) SchedMD LLC. -Copyright 2018 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | ~> 1.0 | -| [google](#requirement\_google) | >= 3.43 | -| [null](#requirement\_null) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.43 | -| [null](#provider\_null) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_instance_from_template.slurm_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_from_template) | resource | -| [null_resource.replace_trigger](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_instance_template.base](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance_template) | data source | -| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    network            = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    queue_count        = optional(number)
    stack_type         = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
  }))
| `[]` | no | -| [hostname](#input\_hostname) | Hostname of instances | `string` | n/a | yes | -| [instance\_template](#input\_instance\_template) | Instance template self\_link used to create compute instances | `string` | n/a | yes | -| [network](#input\_network) | Network to deploy to. Only one of network or subnetwork should be specified. | `string` | `""` | no | -| [num\_instances](#input\_num\_instances) | Number of instances to create. This value is ignored if static\_ips is provided. | `number` | `1` | no | -| [project\_id](#input\_project\_id) | The GCP project ID | `string` | `null` | no | -| [region](#input\_region) | Region where the instances should be created. | `string` | `null` | no | -| [replace\_trigger](#input\_replace\_trigger) | Trigger value to replace the instances. | `string` | `""` | no | -| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances | `list(string)` | `[]` | no | -| [subnetwork](#input\_subnetwork) | Subnet to deploy to. Only one of network or subnetwork should be specified. | `string` | `""` | no | -| [subnetwork\_project](#input\_subnetwork\_project) | The project that subnetwork belongs to | `string` | `null` | no | -| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be spread across available zones in the region. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [available\_zones](#output\_available\_zones) | List of available zones in region | -| [instances\_details](#output\_instances\_details) | List of all details for compute instances | -| [instances\_self\_links](#output\_instances\_self\_links) | List of self-links for compute instances | -| [names](#output\_names) | List of available zones in region | -| [slurm\_instances](#output\_slurm\_instances) | List of all resource objects for compute instances | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf deleted file mode 100644 index 2af9008a0e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/main.tf +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -########## -# LOCALS # -########## - -locals { - num_instances = length(var.static_ips) == 0 ? var.num_instances : length(var.static_ips) - - # local.static_ips is the same as var.static_ips with a dummy element appended - # at the end of the list to work around "list does not have any elements so cannot - # determine type" error when var.static_ips is empty - static_ips = concat(var.static_ips, ["NOT_AN_IP"]) - - network_interfaces = [for index in range(local.num_instances) : - concat([ - { - access_config = var.access_config - alias_ip_range = [] - ipv6_access_config = [] - network = var.network - network_ip = length(var.static_ips) == 0 ? "" : element(local.static_ips, index) - nic_type = null - queue_count = null - stack_type = null - subnetwork = var.subnetwork - subnetwork_project = var.subnetwork_project - } - ], - var.additional_networks - ) - ] -} - -################ -# DATA SOURCES # -################ - -data "google_compute_zones" "available" { - project = var.project_id - region = var.region -} - -data "google_compute_instance_template" "base" { - project = var.project_id - name = var.instance_template -} - -############# -# INSTANCES # -############# -resource "null_resource" "replace_trigger" { - triggers = { - trigger = var.replace_trigger - } -} - -# TODO: `internal/slurm-gcp/login` is ONLY user of `internal/slurm-gcp/instance` -# Remove this module, add functionality (+ prune generality) to the login module directly. -resource "google_compute_instance_from_template" "slurm_instance" { - count = local.num_instances - name = format("%s-%s", var.hostname, format("%03d", count.index + 1)) - project = var.project_id - zone = var.zone == null ? data.google_compute_zones.available.names[count.index % length(data.google_compute_zones.available.names)] : var.zone - - allow_stopping_for_update = true - - dynamic "network_interface" { - for_each = local.network_interfaces[count.index] - iterator = nic - content { - dynamic "access_config" { - for_each = nic.value.access_config - content { - nat_ip = access_config.value.nat_ip - network_tier = access_config.value.network_tier - } - } - dynamic "alias_ip_range" { - for_each = nic.value.alias_ip_range - content { - ip_cidr_range = alias_ip_range.value.ip_cidr_range - subnetwork_range_name = alias_ip_range.value.subnetwork_range_name - } - } - dynamic "ipv6_access_config" { - for_each = nic.value.ipv6_access_config - iterator = access_config - content { - network_tier = access_config.value.network_tier - } - } - network = nic.value.network - network_ip = nic.value.network_ip - nic_type = nic.value.nic_type - queue_count = nic.value.queue_count - subnetwork = nic.value.subnetwork - subnetwork_project = nic.value.subnetwork_project - } - } - - source_instance_template = data.google_compute_instance_template.base.self_link - # Due to https://github.com/hashicorp/terraform-provider-google/issues/21693 - # we have to explicitly override instance labels instead of inheriting them from template. - labels = data.google_compute_instance_template.base.labels - - - lifecycle { - replace_triggered_by = [null_resource.replace_trigger.id] - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf deleted file mode 100644 index 4eba78a7e8..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/outputs.tf +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "slurm_instances" { - description = "List of all resource objects for compute instances" - value = google_compute_instance_from_template.slurm_instance -} - -output "instances_self_links" { - description = "List of self-links for compute instances" - value = google_compute_instance_from_template.slurm_instance[*].self_link -} - -output "instances_details" { - description = "List of all details for compute instances" - value = google_compute_instance_from_template.slurm_instance[*] -} - -output "available_zones" { - description = "List of available zones in region" - value = data.google_compute_zones.available.names -} - -output "names" { - description = "List of available zones in region" - value = google_compute_instance_from_template.slurm_instance[*].name -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf deleted file mode 100644 index 11111a2c05..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/variables.tf +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - type = string - description = "The GCP project ID" - default = null -} - -variable "network" { - description = "Network to deploy to. Only one of network or subnetwork should be specified." - type = string - default = "" -} - -variable "subnetwork" { - description = "Subnet to deploy to. Only one of network or subnetwork should be specified." - type = string - default = "" -} - -variable "subnetwork_project" { - description = "The project that subnetwork belongs to" - type = string - default = null -} - -variable "hostname" { - description = "Hostname of instances" - type = string -} - -variable "additional_networks" { - description = "Additional network interface details for GCE, if any." - default = [] - type = list(object({ - access_config = optional(list(object({ - nat_ip = string - network_tier = string - })), []) - alias_ip_range = optional(list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })), []) - ipv6_access_config = optional(list(object({ - network_tier = string - })), []) - network = optional(string) - network_ip = optional(string, "") - nic_type = optional(string) - queue_count = optional(number) - stack_type = optional(string) - subnetwork = optional(string) - subnetwork_project = optional(string) - })) - nullable = false -} - -variable "static_ips" { - description = "List of static IPs for VM instances" - type = list(string) - default = [] -} - -variable "access_config" { - description = "Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet." - type = list(object({ - nat_ip = string - network_tier = string - })) - default = [] -} - -variable "num_instances" { - description = "Number of instances to create. This value is ignored if static_ips is provided." - type = number - default = 1 -} - -variable "instance_template" { - description = "Instance template self_link used to create compute instances" - type = string -} - -variable "region" { - description = "Region where the instances should be created." - type = string - default = null -} - -variable "zone" { - description = "Zone where the instances should be created. If not specified, instances will be spread across available zones in the region." - type = string - default = null -} - -######### -# SLURM # -######### - -variable "replace_trigger" { - description = "Trigger value to replace the instances." - type = string - default = "" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf deleted file mode 100644 index a3e84c09bf..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance/versions.tf +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = "~> 1.0" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.43" - } - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md deleted file mode 100644 index 8b57f96642..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/README.md +++ /dev/null @@ -1,86 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | ~> 1.0 | -| [local](#requirement\_local) | ~> 2.0 | - -## Providers - -| Name | Version | -|------|---------| -| [local](#provider\_local) | ~> 2.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [instance\_template](#module\_instance\_template) | ../internal_instance_template | n/a | - -## Resources - -| Name | Type | -|------|------| -| [local_file.startup](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | -| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    source                     = optional(string)
    disk_name                  = optional(string)
    device_name                = string
    disk_type                  = optional(string)
    disk_size_gb               = optional(number)
    disk_labels                = map(string)
    auto_delete                = bool
    boot                       = bool
    disk_resource_manager_tags = optional(map(string))
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| n/a | yes | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `virtio_enabled` setting will only enable VirtioNet and will not enable TIER\_1.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image that supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | -| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | -| [disk\_labels](#input\_disk\_labels) | Labels to be assigned to boot disk, provided as a map. | `map(string)` | `{}` | no | -| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `100` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either pd-ssd, local-ssd, or pd-standard. | `string` | `"pd-standard"` | no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [gpu](#input\_gpu) | GPU information. Type and count of GPU to attach to the instance template. See
https://cloud.google.com/compute/docs/gpus more details.
- type : the GPU type
- count : number of GPUs | 
object({
    type  = string
    count = number
  })
| `null` | no | -| [internal\_startup\_script](#input\_internal\_startup\_script) | FOR INTERNAL TOOLKIT USAGE ONLY. | `string` | `null` | no | -| [labels](#input\_labels) | Labels, provided as a map | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"n1-standard-1"` | no | -| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | -| [name\_prefix](#input\_name\_prefix) | Prefix for template resource. | `string` | `"default"` | no | -| [network](#input\_network) | The name or self\_link of the network to attach this interface to. Use network
attribute for Legacy or Auto subnetted networks and subnetwork for custom
subnetted networks. | `string` | `null` | no | -| [network\_ip](#input\_network\_ip) | Private IP address to assign to the instance if desired. | `string` | `""` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy | `string` | `"MIGRATE"` | no | -| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [provisioning\_model](#input\_provisioning\_model) | The provisioning model of the instance | `string` | `null` | no | -| [region](#input\_region) | Region where the instance template should be created. | `string` | n/a | yes | -| [reservation\_affinity](#input\_reservation\_affinity) | Specifies the reservations that this instance can consume from. | `object({ type = string })` | `null` | no | -| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [service\_account](#input\_service\_account) | Service account to attach to the instances. See
'main.tf:local.service\_account' for the default. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used for resource naming. | `string` | n/a | yes | -| [slurm\_instance\_role](#input\_slurm\_instance\_role) | Slurm instance type. Must be one of: controller; login; compute; or null. | `string` | n/a | yes | -| [source\_image](#input\_source\_image) | Source disk image. | `string` | `""` | no | -| [source\_image\_family](#input\_source\_image\_family) | Source image family. | `string` | `""` | no | -| [source\_image\_project](#input\_source\_image\_project) | Project where the source image comes from. If it is not provided, the provider project is used. | `string` | `""` | no | -| [spot](#input\_spot) | Provision as a SPOT preemptible instance.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `bool` | `false` | no | -| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach this interface to. The subnetwork must
exist in the same region this instance will be created in. Either network or
subnetwork must be provided. | `string` | `null` | no | -| [subnetwork\_project](#input\_subnetwork\_project) | The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used. | `string` | `null` | no | -| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | -| [termination\_action](#input\_termination\_action) | Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instance\_template](#output\_instance\_template) | Instance template details | -| [labels](#output\_labels) | Labels attached to the instance template | -| [name](#output\_name) | Name of instance template | -| [self\_link](#output\_self\_link) | Self\_link of instance template | -| [service\_account](#output\_service\_account) | Service account object, includes email and scopes. | -| [tags](#output\_tags) | Tags that will be associated with instance(s) | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted deleted file mode 100644 index fb6191eec6..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/files/startup_sh_unlinted +++ /dev/null @@ -1,161 +0,0 @@ -#!/bin/bash -# Copyright (C) SchedMD LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -SLURM_DIR=/slurm -FLAGFILE=$SLURM_DIR/slurm_configured_do_not_remove -SCRIPTS_DIR=$SLURM_DIR/scripts -if [[ -z "$HOME" ]]; then - # google-startup-scripts.service lacks environment variables - HOME="$(getent passwd "$(whoami)" | cut -d: -f6)" -fi - -# Temporary workaround for transition period when some of older images -# don't have "baked in" python yet. -# TODO: Remove -SLURM_PY="/slurm/python/venv/bin/python3.13" -SYSTEM_PY="/usr/bin/python3" -if [[ ! -e "$SLURM_PY" ]]; then - echo "Symlink $SLURM_PY does not exist. Creating symlink to $SYSTEM_PY" - mkdir -p /slurm/python/venv/bin - ln -s "$SYSTEM_PY" "$SLURM_PY" -fi - -METADATA_SERVER="metadata.google.internal" -URL="http://$METADATA_SERVER/computeMetadata/v1" -CURL="curl -sS --fail --header Metadata-Flavor:Google" - -PING_METADATA="ping -q -w1 -c1 $METADATA_SERVER" -echo "INFO: $PING_METADATA" -for i in $(seq 10); do - [ $i -gt 1 ] && sleep 5; - $PING_METADATA > /dev/null && s=0 && break || s=$?; - echo "ERROR: Failed to contact metadata server, will retry" -done -if [ $s -ne 0 ]; then - echo "ERROR: Unable to contact metadata server, aborting" - wall -n '*** Slurm setup failed in the startup script! see `journalctl -u google-startup-scripts` ***' - exit 1 -else - echo "INFO: Successfully contacted metadata server" -fi - -PING_GOOGLE="ping -q -w1 -c1 8.8.8.8" -echo "INFO: $PING_GOOGLE" -for i in $(seq 5); do - [ $i -gt 1 ] && sleep 2; - $PING_GOOGLE > /dev/null && s=0 && break || s=$?; - echo "failed to ping Google DNS, will retry" -done -if [ $s -ne 0 ]; then - echo "WARNING: No internet access detected" -else - echo "INFO: Internet access detected" -fi - -mkdir -p $SCRIPTS_DIR -UNIVERSE_DOMAIN="$($CURL $URL/instance/attributes/universe_domain)" -BUCKET="$($CURL $URL/instance/attributes/slurm_bucket_path)" -if [[ -z $BUCKET ]]; then - echo "ERROR: No bucket path detected." - exit 1 -fi - -SCRIPTS_ZIP="$HOME/slurm-gcp-scripts.zip" -export CLOUDSDK_CORE_UNIVERSE_DOMAIN="$UNIVERSE_DOMAIN" -until gcloud storage cp "$BUCKET/slurm-gcp-devel.zip" "$SCRIPTS_ZIP"; do - echo "WARN: Could not download SlurmGCP scripts, retrying in 5 seconds." - # Remove marker used to determine if gcloud is being used in a GCE VM. - # This can get mistakenly set to False in some cases. - rm -f /root/.config/gcloud/gce - sleep 5 -done -unzip -o "$SCRIPTS_ZIP" -d "$SCRIPTS_DIR" -rm -rf "$SCRIPTS_ZIP" - -#temporary hack to not make the script fail on TPU vm -chown slurm:slurm -R "$SCRIPTS_DIR" || true -chmod 700 -R "$SCRIPTS_DIR" - - -if [ -f $FLAGFILE ]; then - echo "WARNING: Slurm was previously configured, quitting" - exit 0 -fi -touch $FLAGFILE - -function tpu_setup { - #allow the following command to fail, as this attribute does not exist for regular nodes - docker_image=$($CURL $URL/instance/attributes/slurm_docker_image 2> /dev/null || true) - if [ -z $docker_image ]; then #Not a tpu node, do not do anything - return - fi - if [ "$OS_ENV" == "slurm_container" ]; then #Already inside the slurm container, we should continue starting - return - fi - - #given a input_string like "WORKER_0:Joseph;WORKER_1:richard;WORKER_2:edward;WORKER_3:john" and a number 1, this function will print richard - parse_metadata() { - local number=$1 - local input_string=$2 - local word=$(echo "$input_string" | awk -v n="$number" -F ':|;' '{ for (i = 1; i <= NF; i+=2) if ($(i) == "WORKER_"n) print $(i+1) }') - echo "$word" - } - - input_string=$($CURL $URL/instance/attributes/slurm_names) - worker_id=$($CURL $URL/instance/attributes/tpu-env | awk '/WORKER_ID/ {print $2}' | tr -d \') - real_name=$(parse_metadata $worker_id $input_string) - - #Prepare to docker pull with gcloud - mkdir -p /root/.docker - cat << EOF > /root/.docker/config.json -{ - "credHelpers": { - "gcr.io": "gcloud", - "us-docker.pkg.dev": "gcloud" - } -} -EOF - #cgroup detection - CGV=1 - CGROUP_FLAGS="-v /sys/fs/cgroup:/sys/fs/cgroup:rw" - if [ -f /sys/fs/cgroup/cgroup.controllers ]; then #CGV2 - CGV=2 - fi - if [ $CGV == 2 ]; then - CGROUP_FLAGS="--cgroup-parent=docker.slice --cgroupns=private --tmpfs /run --tmpfs /run/lock --tmpfs /tmp" - if [ ! -f /etc/systemd/system/docker.slice ]; then #In case that there is no slice prepared for hosting the containers create it - printf "[Unit]\nDescription=docker slice\nBefore=slices.target\n[Slice]\nCPUAccounting=true\nMemoryAccounting=true" > /etc/systemd/system/docker.slice - systemctl start docker.slice - fi - fi - #for the moment always use --privileged, as systemd might not work properly otherwise - TPU_FLAGS="--privileged" - # TPU_FLAGS="--cap-add SYS_RESOURCE --device /dev/accel0 --device /dev/accel1 --device /dev/accel2 --device /dev/accel3" - # if [ $CGV == 2 ]; then #In case that we are in CGV2 for systemd to work correctly for the moment we go with privileged - # TPU_FLAGS="--privileged" - # fi - - docker run -d $CGROUP_FLAGS $TPU_FLAGS --net=host --name=slurmd --hostname=$real_name --entrypoint=/usr/bin/systemd --restart unless-stopped $docker_image - exit 0 -} - -tpu_setup #will do nothing for normal nodes or the container spawned inside TPU - -echo "INFO: Running python cluster setup script" -SETUP_SCRIPT_FILE=$SCRIPTS_DIR/setup.py -chmod +x $SETUP_SCRIPT_FILE -exec $SETUP_SCRIPT_FILE diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf deleted file mode 100644 index 3ec9f466de..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/main.tf +++ /dev/null @@ -1,164 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -########## -# LOCALS # -########## - -locals { - additional_disks = [ - for disk in var.additional_disks : { - disk_name = disk.disk_name - device_name = disk.device_name - auto_delete = disk.auto_delete - source = disk.source - boot = disk.boot - disk_size_gb = disk.disk_size_gb - disk_type = disk.disk_type - disk_labels = merge( - disk.disk_labels, - { - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = var.slurm_instance_role - }, - ) - disk_resource_manager_tags = disk.disk_resource_manager_tags - } - ] - - service_account = { - email = try(var.service_account.email, null) - scopes = try(var.service_account.scopes, ["https://www.googleapis.com/auth/cloud-platform"]) - } - - source_image_family = ( - var.source_image_family != "" && var.source_image_family != null - ? var.source_image_family - : "slurm-gcp-6-9-hpc-rocky-linux-8" - ) - source_image_project = ( - var.source_image_project != "" && var.source_image_project != null - ? var.source_image_project - : "projects/schedmd-slurm-public/global/images/family" - ) - - source_image = ( - var.source_image != null - ? var.source_image - : "" - ) - - - name_prefix = "${var.slurm_cluster_name}-${var.slurm_instance_role}-${var.name_prefix}" - - total_egress_bandwidth_tier = var.bandwidth_tier == "tier_1_enabled" ? "TIER_1" : "DEFAULT" - - nic_type_map = { - platform_default = null - virtio_enabled = "VIRTIO_NET" - gvnic_enabled = "GVNIC" - tier_1_enabled = "GVNIC" - } - nic_type = lookup(local.nic_type_map, var.bandwidth_tier, null) - - labels = merge(var.labels, - { - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = var.slurm_instance_role - }, - ) -} - -######## -# DATA # -######## - -data "local_file" "startup" { - filename = "${path.module}/files/startup_sh_unlinted" -} - -############ -# TEMPLATE # -############ - -module "instance_template" { - source = "../internal_instance_template" - - project_id = var.project_id - - # Network - can_ip_forward = var.can_ip_forward - network_ip = var.network_ip - network = var.network - nic_type = local.nic_type - region = var.region - subnetwork_project = var.subnetwork_project - subnetwork = var.subnetwork - tags = var.tags - total_egress_bandwidth_tier = local.total_egress_bandwidth_tier - additional_networks = var.additional_networks - access_config = var.access_config - - # Instance - machine_type = var.machine_type - min_cpu_platform = var.min_cpu_platform - name_prefix = local.name_prefix - gpu = var.gpu - service_account = local.service_account - shielded_instance_config = var.shielded_instance_config - advanced_machine_features = var.advanced_machine_features - enable_confidential_vm = var.enable_confidential_vm - enable_shielded_vm = var.enable_shielded_vm - preemptible = var.preemptible - spot = var.spot - on_host_maintenance = var.on_host_maintenance - labels = local.labels - instance_termination_action = var.termination_action - resource_manager_tags = var.resource_manager_tags - - # Metadata - startup_script = coalesce(var.internal_startup_script, data.local_file.startup.content) - metadata = merge( - var.metadata, - { - enable-oslogin = upper(var.enable_oslogin) - slurm_bucket_path = var.slurm_bucket_path - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = var.slurm_instance_role - }, - ) - - # Image - source_image_project = local.source_image_project - source_image_family = local.source_image_family - source_image = local.source_image - - # Disk - disk_type = var.disk_type - disk_size_gb = var.disk_size_gb - auto_delete = var.disk_auto_delete - disk_labels = merge( - { - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = var.slurm_instance_role - }, - var.disk_labels, - ) - disk_resource_manager_tags = var.disk_resource_manager_tags - additional_disks = local.additional_disks - - max_run_duration = var.max_run_duration - provisioning_model = var.provisioning_model - reservation_affinity = var.reservation_affinity -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf deleted file mode 100644 index 65da41052e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/outputs.tf +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "instance_template" { - description = "Instance template details" - value = module.instance_template -} - -output "self_link" { - description = "Self_link of instance template" - value = module.instance_template.self_link -} - -output "name" { - description = "Name of instance template" - value = module.instance_template.name -} - -output "tags" { - description = "Tags that will be associated with instance(s)" - value = module.instance_template.tags -} - -output "service_account" { - description = "Service account object, includes email and scopes." - value = module.instance_template.service_account -} - -output "labels" { - description = "Labels attached to the instance template" - value = local.labels -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf deleted file mode 100644 index 35dd9c376f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/instance_template/variables.tf +++ /dev/null @@ -1,431 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -########### -# GENERAL # -########### - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "on_host_maintenance" { - type = string - description = "Instance availability Policy" - default = "MIGRATE" -} - -variable "labels" { - type = map(string) - description = "Labels, provided as a map" - default = {} -} - -variable "enable_oslogin" { - type = bool - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >=0.13.0 | -| [google](#requirement\_google) | >= 3.88 | -| [google-beta](#requirement\_google-beta) | >= 6.13.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.88 | -| [google-beta](#provider\_google-beta) | >= 6.13.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_instance_template.tpl](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance_template) | resource | -| [google_project.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_config](#input\_access\_config) | Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet. | 
list(object({
    nat_ip       = string
    network_tier = string
  }))
| `[]` | no | -| [additional\_disks](#input\_additional\_disks) | List of maps of additional disks. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#disk_name | 
list(object({
    source                     = optional(string)
    disk_name                  = optional(string)
    device_name                = string
    auto_delete                = bool
    boot                       = bool
    disk_size_gb               = optional(number)
    disk_type                  = optional(string)
    disk_labels                = map(string)
    disk_resource_manager_tags = map(string)
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| n/a | yes | -| [alias\_ip\_range](#input\_alias\_ip\_range) | An array of alias IP ranges for this network interface. Can only be specified for network interfaces on subnet-mode networks.
ip\_cidr\_range: The IP CIDR range represented by this alias IP range. This IP CIDR range must belong to the specified subnetwork and cannot contain IP addresses reserved by system or used by other network interfaces. At the time of writing only a netmask (e.g. /24) may be supplied, with a CIDR format resulting in an API error.
subnetwork\_range\_name: The subnetwork secondary range name specifying the secondary range from which to allocate the IP CIDR range for this alias IP range. If left unspecified, the primary range of the subnetwork will be used. | 
object({
    ip_cidr_range         = string
    subnetwork_range_name = string
  })
| `null` | no | -| [auto\_delete](#input\_auto\_delete) | Whether or not the boot disk should be auto-deleted | `string` | `"true"` | no | -| [automatic\_restart](#input\_automatic\_restart) | (Optional) Specifies whether the instance should be automatically restarted if it is terminated by Compute Engine (not terminated by a user). | `bool` | `true` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example | `string` | `"false"` | no | -| [disk\_encryption\_key](#input\_disk\_encryption\_key) | The id of the encryption key that is stored in Google Cloud KMS to use to encrypt all the disks on this instance | `string` | `null` | no | -| [disk\_labels](#input\_disk\_labels) | Labels to be assigned to boot disk, provided as a map | `map(string)` | `{}` | no | -| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `string` | `"100"` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either pd-ssd, local-ssd, or pd-standard | `string` | `"pd-standard"` | no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Whether to enable the Confidential VM configuration on the instance. Note that the instance image must support Confidential VMs. See https://cloud.google.com/compute/docs/images | `bool` | `false` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Whether to enable the Shielded VM configuration on the instance. Note that the instance image must support Shielded VMs. See https://cloud.google.com/compute/docs/images | `bool` | `false` | no | -| [gpu](#input\_gpu) | GPU information. Type and count of GPU to attach to the instance template. See https://cloud.google.com/compute/docs/gpus more details | 
object({
    type  = string
    count = number
  })
| `null` | no | -| [instance\_termination\_action](#input\_instance\_termination\_action) | Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `string` | `null` | no | -| [ipv6\_access\_config](#input\_ipv6\_access\_config) | IPv6 access configurations. Currently a max of 1 IPv6 access configuration is supported. If not specified, the instance will have no external IPv6 Internet access. | 
list(object({
    network_tier = string
  }))
| `[]` | no | -| [labels](#input\_labels) | Labels, provided as a map | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Machine type to create, e.g. n1-standard-1 | `string` | `"n1-standard-1"` | no | -| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list: https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | -| [name\_prefix](#input\_name\_prefix) | Name prefix for the instance template | `string` | n/a | yes | -| [network](#input\_network) | The name or self\_link of the network to attach this interface to. Use network attribute for Legacy or Auto subnetted networks and subnetwork for custom subnetted networks. | `string` | `""` | no | -| [network\_ip](#input\_network\_ip) | Private IP address to assign to the instance if desired. | `string` | `""` | no | -| [nic\_type](#input\_nic\_type) | The type of vNIC to be used on this interface. Possible values: GVNIC, VIRTIO\_NET. | `string` | `null` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy | `string` | `"MIGRATE"` | no | -| [preemptible](#input\_preemptible) | Allow the instance to be preempted | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | The GCP project ID | `string` | `null` | no | -| [provisioning\_model](#input\_provisioning\_model) | The provisioning model of the instance | `string` | `null` | no | -| [region](#input\_region) | Region where the instance template should be created. | `string` | n/a | yes | -| [reservation\_affinity](#input\_reservation\_affinity) | Specifies the reservations that this instance can consume from. | `object({ type = string })` | `null` | no | -| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [service\_account](#input\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#service_account. | 
object({
    email  = optional(string)
    scopes = set(string)
  })
| n/a | yes | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Not used unless enable\_shielded\_vm is true. Shielded VM configuration for the instance. | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [source\_image](#input\_source\_image) | Source disk image. If neither source\_image nor source\_image\_family is specified, defaults to the latest public CentOS image. | `string` | `""` | no | -| [source\_image\_family](#input\_source\_image\_family) | Source image family. If neither source\_image nor source\_image\_family is specified, defaults to the latest public CentOS image. | `string` | `"centos-7"` | no | -| [source\_image\_project](#input\_source\_image\_project) | Project where the source image comes from. The default project contains CentOS images. | `string` | `"centos-cloud"` | no | -| [spot](#input\_spot) | Provision as a SPOT preemptible instance.
See https://cloud.google.com/compute/docs/instances/spot for more details. | `bool` | `false` | no | -| [stack\_type](#input\_stack\_type) | The stack type for this network interface to identify whether the IPv6 feature is enabled or not. Values are `IPV4_IPV6` or `IPV4_ONLY`. Default behavior is equivalent to IPV4\_ONLY. | `string` | `null` | no | -| [startup\_script](#input\_startup\_script) | User startup script to run when instances spin up | `string` | `""` | no | -| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach this interface to. The subnetwork must exist in the same region this instance will be created in. Either network or subnetwork must be provided. | `string` | `""` | no | -| [subnetwork\_project](#input\_subnetwork\_project) | The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used. | `string` | `null` | no | -| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | -| [total\_egress\_bandwidth\_tier](#input\_total\_egress\_bandwidth\_tier) | Network bandwidth tier. Note: machine\_type must be a supported type. Values are 'TIER\_1' or 'DEFAULT'.
See https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration for details. | `string` | `"DEFAULT"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [name](#output\_name) | Name of instance template | -| [self\_link](#output\_self\_link) | Self-link of instance template | -| [service\_account](#output\_service\_account) | value | -| [tags](#output\_tags) | Tags that will be associated with instance(s) | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf deleted file mode 100644 index cf69a3f926..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/main.tf +++ /dev/null @@ -1,227 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -######### -# Locals -######### - -locals { - source_image = var.source_image != "" ? var.source_image : "centos-7-v20201112" - source_image_family = var.source_image_family != "" ? var.source_image_family : "centos-7" - source_image_project = var.source_image_project != "" ? var.source_image_project : "centos-cloud" - - boot_disk = [ - { - source_image = var.source_image != "" ? format("${local.source_image_project}/${local.source_image}") : format("${local.source_image_project}/${local.source_image_family}") - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - disk_labels = var.disk_labels - auto_delete = var.auto_delete - disk_resource_manager_tags = var.disk_resource_manager_tags - boot = "true" - }, - ] - - all_disks = concat(local.boot_disk, var.additional_disks) - - # NOTE: Even if all the shielded_instance_config or confidential_instance_config - # values are false, if the config block exists and an unsupported image is chosen, - # the apply will fail so we use a single-value array with the default value to - # initialize the block only if it is enabled. - shielded_vm_configs = var.enable_shielded_vm ? [true] : [] - - gpu_enabled = var.gpu != null - alias_ip_range_enabled = var.alias_ip_range != null - preemptible = var.preemptible || var.spot - on_host_maintenance = ( - local.preemptible || var.enable_confidential_vm || local.gpu_enabled - ? "TERMINATE" - : var.on_host_maintenance - ) - automatic_restart = ( - # must be false when preemptible is true - local.preemptible ? false : var.automatic_restart - ) - - nic_type = var.total_egress_bandwidth_tier == "TIER_1" ? "GVNIC" : var.nic_type - - - provisioning_model = coalesce(var.provisioning_model, local.preemptible ? "SPOT" : "STANDARD") -} - -data "google_project" "this" { - project_id = var.project_id -} - -#################### -# Instance Template -#################### -resource "google_compute_instance_template" "tpl" { - provider = google-beta - name_prefix = "${var.name_prefix}-" - project = var.project_id - machine_type = var.machine_type - labels = var.labels - metadata = var.metadata - tags = var.tags - can_ip_forward = var.can_ip_forward - metadata_startup_script = var.startup_script - region = var.region - min_cpu_platform = var.min_cpu_platform - resource_manager_tags = var.resource_manager_tags - - service_account { - email = coalesce(var.service_account.email, "${data.google_project.this.number}-compute@developer.gserviceaccount.com") - scopes = lookup(var.service_account, "scopes", null) - } - - dynamic "disk" { - for_each = local.all_disks - content { - auto_delete = lookup(disk.value, "auto_delete", null) - boot = lookup(disk.value, "boot", null) - device_name = lookup(disk.value, "device_name", null) - disk_name = lookup(disk.value, "disk_name", null) - disk_size_gb = lookup(disk.value, "disk_size_gb", lookup(disk.value, "disk_type", null) == "local-ssd" ? "375" : null) - disk_type = lookup(disk.value, "disk_type", null) - interface = lookup(disk.value, "interface", lookup(disk.value, "disk_type", null) == "local-ssd" ? "NVME" : null) - mode = lookup(disk.value, "mode", null) - source = lookup(disk.value, "source", null) - source_image = lookup(disk.value, "source_image", null) - type = lookup(disk.value, "disk_type", null) == "local-ssd" ? "SCRATCH" : "PERSISTENT" - labels = (lookup(disk.value, "source", null) != null || lookup(disk.value, "disk_type", null) == "local-ssd") ? null : lookup(disk.value, "disk_labels", null) - resource_manager_tags = lookup(disk.value, "disk_resource_manager_tags", {}) - - dynamic "disk_encryption_key" { - for_each = compact([var.disk_encryption_key == null ? null : 1]) - content { - kms_key_self_link = var.disk_encryption_key - } - } - } - } - - network_interface { - network = var.network - subnetwork = var.subnetwork - subnetwork_project = var.subnetwork_project - network_ip = try(coalesce(var.network_ip), null) - nic_type = local.nic_type - stack_type = var.stack_type - dynamic "access_config" { - for_each = var.access_config - content { - nat_ip = access_config.value.nat_ip - network_tier = access_config.value.network_tier - } - } - dynamic "ipv6_access_config" { - for_each = var.ipv6_access_config - content { - network_tier = ipv6_access_config.value.network_tier - } - } - dynamic "alias_ip_range" { - for_each = local.alias_ip_range_enabled ? [var.alias_ip_range] : [] - content { - ip_cidr_range = alias_ip_range.value.ip_cidr_range - subnetwork_range_name = alias_ip_range.value.subnetwork_range_name - } - } - } - - dynamic "network_interface" { - for_each = var.additional_networks - content { - network = network_interface.value.network - subnetwork = network_interface.value.subnetwork - subnetwork_project = network_interface.value.subnetwork_project - network_ip = try(coalesce(network_interface.value.network_ip), null) - nic_type = try(coalesce(network_interface.value.nic_type), null) - dynamic "access_config" { - for_each = network_interface.value.access_config - content { - nat_ip = access_config.value.nat_ip - network_tier = access_config.value.network_tier - } - } - dynamic "ipv6_access_config" { - for_each = network_interface.value.ipv6_access_config - content { - network_tier = ipv6_access_config.value.network_tier - } - } - } - } - - network_performance_config { - total_egress_bandwidth_tier = coalesce(var.total_egress_bandwidth_tier, "DEFAULT") - } - - lifecycle { - create_before_destroy = "true" - } - - scheduling { - preemptible = local.preemptible - provisioning_model = local.provisioning_model - automatic_restart = local.automatic_restart - on_host_maintenance = local.on_host_maintenance - instance_termination_action = var.instance_termination_action - - dynamic "max_run_duration" { - for_each = var.max_run_duration != null ? [var.max_run_duration] : [] - content { - seconds = max_run_duration.value - } - } - } - - dynamic "reservation_affinity" { - for_each = var.reservation_affinity != null ? [var.reservation_affinity] : [] - content { - type = reservation_affinity.value.type - } - } - - advanced_machine_features { - enable_nested_virtualization = var.advanced_machine_features.enable_nested_virtualization - threads_per_core = var.advanced_machine_features.threads_per_core - turbo_mode = var.advanced_machine_features.turbo_mode - visible_core_count = var.advanced_machine_features.visible_core_count - performance_monitoring_unit = var.advanced_machine_features.performance_monitoring_unit - enable_uefi_networking = var.advanced_machine_features.enable_uefi_networking - } - - dynamic "shielded_instance_config" { - for_each = local.shielded_vm_configs - content { - enable_secure_boot = lookup(var.shielded_instance_config, "enable_secure_boot", shielded_instance_config.value) - enable_vtpm = lookup(var.shielded_instance_config, "enable_vtpm", shielded_instance_config.value) - enable_integrity_monitoring = lookup(var.shielded_instance_config, "enable_integrity_monitoring", shielded_instance_config.value) - } - } - - confidential_instance_config { - enable_confidential_compute = var.enable_confidential_vm - } - - dynamic "guest_accelerator" { - for_each = local.gpu_enabled ? [var.gpu] : [] - content { - type = guest_accelerator.value.type - count = guest_accelerator.value.count - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf deleted file mode 100644 index 69f8d3b98c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/outputs.tf +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "self_link" { - description = "Self-link of instance template" - value = google_compute_instance_template.tpl.self_link -} - -output "name" { - description = "Name of instance template" - value = google_compute_instance_template.tpl.name -} - -output "tags" { - description = "Tags that will be associated with instance(s)" - value = google_compute_instance_template.tpl.tags -} - -output "service_account" { - description = "value" - value = google_compute_instance_template.tpl.service_account[0] -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf deleted file mode 100644 index c285c3fea5..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/internal_instance_template/variables.tf +++ /dev/null @@ -1,398 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - type = string - description = "The GCP project ID" - default = null -} - -variable "name_prefix" { - description = "Name prefix for the instance template" - type = string -} - -variable "machine_type" { - description = "Machine type to create, e.g. n1-standard-1" - type = string - default = "n1-standard-1" -} - -variable "min_cpu_platform" { - description = "Specifies a minimum CPU platform. Applicable values are the friendly names of CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list: https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform" - type = string - default = null -} - -variable "can_ip_forward" { - description = "Enable IP forwarding, for NAT instances for example" - type = string - default = "false" -} - -variable "tags" { - type = list(string) - description = "Network tags, provided as a list" - default = [] -} - -variable "labels" { - type = map(string) - description = "Labels, provided as a map" - default = {} -} - -variable "preemptible" { - type = bool - description = "Allow the instance to be preempted" - default = false -} - -variable "spot" { - description = <<-EOD - Provision as a SPOT preemptible instance. - See https://cloud.google.com/compute/docs/instances/spot for more details. - EOD - type = bool - default = false -} - -variable "instance_termination_action" { - description = <<-EOD - Which action to take when Compute Engine preempts the VM. Value can be: 'STOP', 'DELETE'. The default value is 'STOP'. - See https://cloud.google.com/compute/docs/instances/spot for more details. - EOD - type = string - default = null -} - -variable "automatic_restart" { - type = bool - description = "(Optional) Specifies whether the instance should be automatically restarted if it is terminated by Compute Engine (not terminated by a user)." - default = true -} - -variable "on_host_maintenance" { - type = string - description = "Instance availability Policy" - default = "MIGRATE" -} - -variable "region" { - type = string - description = "Region where the instance template should be created." - nullable = false -} - -variable "advanced_machine_features" { - description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" - type = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) -} - -variable "resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} - validation { - condition = alltrue([for value in var.resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) - error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" - } - validation { - condition = alltrue([for value in keys(var.resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) - error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" - } -} - -####### -# disk -####### -variable "source_image" { - description = "Source disk image. If neither source_image nor source_image_family is specified, defaults to the latest public CentOS image." - type = string - default = "" -} - -variable "source_image_family" { - description = "Source image family. If neither source_image nor source_image_family is specified, defaults to the latest public CentOS image." - type = string - default = "centos-7" -} - -variable "source_image_project" { - description = "Project where the source image comes from. The default project contains CentOS images." - type = string - default = "centos-cloud" -} - -variable "disk_size_gb" { - description = "Boot disk size in GB" - type = string - default = "100" -} - -variable "disk_type" { - description = "Boot disk type, can be either pd-ssd, local-ssd, or pd-standard" - type = string - default = "pd-standard" -} - -variable "disk_labels" { - description = "Labels to be assigned to boot disk, provided as a map" - type = map(string) - default = {} -} - -variable "disk_encryption_key" { - description = "The id of the encryption key that is stored in Google Cloud KMS to use to encrypt all the disks on this instance" - type = string - default = null -} - -variable "auto_delete" { - description = "Whether or not the boot disk should be auto-deleted" - type = string - default = "true" -} - -variable "disk_resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} - validation { - condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) - error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" - } - validation { - condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) - error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" - } -} - -variable "additional_disks" { - description = "List of maps of additional disks. See https://www.terraform.io/docs/providers/google/r/compute_instance_template#disk_name" - type = list(object({ - source = optional(string) - disk_name = optional(string) - device_name = string - auto_delete = bool - boot = bool - disk_size_gb = optional(number) - disk_type = optional(string) - disk_labels = map(string) - disk_resource_manager_tags = map(string) - })) - default = [] -} - -#################### -# network_interface -#################### -variable "network" { - description = "The name or self_link of the network to attach this interface to. Use network attribute for Legacy or Auto subnetted networks and subnetwork for custom subnetted networks." - type = string - default = "" -} - -variable "nic_type" { - description = "The type of vNIC to be used on this interface. Possible values: GVNIC, VIRTIO_NET." - type = string - default = null -} - -variable "subnetwork" { - description = "The name of the subnetwork to attach this interface to. The subnetwork must exist in the same region this instance will be created in. Either network or subnetwork must be provided." - type = string - default = "" -} - -variable "subnetwork_project" { - description = "The ID of the project in which the subnetwork belongs. If it is not provided, the provider project is used." - type = string - default = null -} - -variable "network_ip" { - description = "Private IP address to assign to the instance if desired." - type = string - default = "" -} - -variable "stack_type" { - description = "The stack type for this network interface to identify whether the IPv6 feature is enabled or not. Values are `IPV4_IPV6` or `IPV4_ONLY`. Default behavior is equivalent to IPV4_ONLY." - type = string - default = null -} - -variable "additional_networks" { - description = "Additional network interface details for GCE, if any." - default = [] - type = list(object({ - network = string - subnetwork = string - subnetwork_project = string - network_ip = string - nic_type = string - access_config = list(object({ - nat_ip = string - network_tier = string - })) - ipv6_access_config = list(object({ - network_tier = string - })) - })) -} - -variable "total_egress_bandwidth_tier" { - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 4.84 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.84 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [instance](#module\_instance) | ../instance | n/a | -| [template](#module\_template) | ../instance_template | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [internal\_startup\_script](#input\_internal\_startup\_script) | FOR INTERNAL TOOLKIT USAGE ONLY. | `string` | `null` | no | -| [login\_nodes](#input\_login\_nodes) | Slurm login instance definitions. | 
object({
    group_name = string
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    additional_networks = optional(list(object({
      access_config = optional(list(object({
        nat_ip       = string
        network_tier = string
      })), [])
      alias_ip_range = optional(list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      })), [])
      ipv6_access_config = optional(list(object({
        network_tier = string
      })), [])
      network            = optional(string)
      network_ip         = optional(string, "")
      nic_type           = optional(string)
      queue_count        = optional(number)
      stack_type         = optional(string)
      subnetwork         = optional(string)
      subnetwork_project = optional(string)
    })), [])
    bandwidth_tier             = optional(string, "platform_default")
    can_ip_forward             = optional(bool, false)
    disk_auto_delete           = optional(bool, true)
    disk_labels                = optional(map(string), {})
    disk_resource_manager_tags = optional(map(string), {})
    disk_size_gb               = optional(number)
    disk_type                  = optional(string, "n1-standard-1")
    enable_confidential_vm     = optional(bool, false)
    enable_oslogin             = optional(bool, true)
    enable_shielded_vm         = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    metadata              = optional(map(string), {})
    min_cpu_platform      = optional(string)
    num_instances         = optional(number, 1)
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    static_ips           = optional(list(string), [])
    subnetwork           = string
    spot                 = optional(bool, false)
    tags                 = optional(list(string), [])
    zone                 = optional(string)
    termination_action   = optional(string)
  })
| n/a | yes | -| [network\_storage](#input\_network\_storage) | Storage to mounted on login instances
- server\_ip : Address of the storage server.
- remote\_mount : The location in the remote instance filesystem to mount from.
- local\_mount : The location on the instance filesystem to mount to.
- fs\_type : Filesystem type (e.g. "nfs").
- mount\_options : Options to mount with. | 
list(object({
    server_ip     = string
    remote_mount  = string
    local_mount   = string
    fs_type       = string
    mount_options = string
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [replace\_trigger](#input\_replace\_trigger) | Trigger value to replace the instances. | `string` | `""` | no | -| [slurm\_bucket\_dir](#input\_slurm\_bucket\_dir) | Path to directory in the bucket for configs | `string` | n/a | yes | -| [slurm\_bucket\_name](#input\_slurm\_bucket\_name) | Name of the bucket for configs | `string` | n/a | yes | -| [slurm\_bucket\_path](#input\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name | `string` | n/a | yes | -| [startup\_scripts](#input\_startup\_scripts) | List of scripts to be ran on login VMs startup. | 
list(object({
    filename = string
    content  = string
  }))
| `[]` | no | -| [startup\_scripts\_timeout](#input\_startup\_scripts\_timeout) | The timeout (seconds) applied to each startup script. If any script exceeds this timeout,
then the instance setup process is considered failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instances](#output\_instances) | VM instances of login nodes | -| [service\_account](#output\_service\_account) | Service Account used by login VMs | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf deleted file mode 100644 index 8185f71fd6..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/main.tf +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -module "template" { - source = "../instance_template" - - project_id = var.project_id - slurm_cluster_name = var.slurm_cluster_name - slurm_instance_role = "login" - slurm_bucket_path = var.slurm_bucket_path - name_prefix = local.name - - additional_disks = var.login_nodes.additional_disks - bandwidth_tier = var.login_nodes.bandwidth_tier - can_ip_forward = var.login_nodes.can_ip_forward - advanced_machine_features = var.login_nodes.advanced_machine_features - disk_auto_delete = var.login_nodes.disk_auto_delete - disk_labels = var.login_nodes.disk_labels - disk_resource_manager_tags = var.login_nodes.disk_resource_manager_tags - disk_size_gb = var.login_nodes.disk_size_gb - disk_type = var.login_nodes.disk_type - enable_confidential_vm = var.login_nodes.enable_confidential_vm - enable_oslogin = var.login_nodes.enable_oslogin - enable_shielded_vm = var.login_nodes.enable_shielded_vm - gpu = var.login_nodes.gpu - labels = var.login_nodes.labels - machine_type = var.login_nodes.machine_type - metadata = merge(var.login_nodes.metadata, { - "universe_domain" = var.universe_domain, - "slurm_login_group" = local.name - }) - min_cpu_platform = var.login_nodes.min_cpu_platform - on_host_maintenance = var.login_nodes.on_host_maintenance - preemptible = var.login_nodes.preemptible - region = var.login_nodes.region - resource_manager_tags = var.login_nodes.resource_manager_tags - service_account = var.login_nodes.service_account - shielded_instance_config = var.login_nodes.shielded_instance_config - source_image_family = var.login_nodes.source_image_family - source_image_project = var.login_nodes.source_image_project - source_image = var.login_nodes.source_image - spot = var.login_nodes.spot - subnetwork = var.login_nodes.subnetwork - tags = concat([var.slurm_cluster_name], var.login_nodes.tags) - termination_action = var.login_nodes.termination_action - - internal_startup_script = var.internal_startup_script -} - -module "instance" { - source = "../instance" - - access_config = var.login_nodes.access_config - hostname = "${var.slurm_cluster_name}-${local.name}" - - project_id = var.project_id - - instance_template = module.template.self_link - num_instances = var.login_nodes.num_instances - - additional_networks = var.login_nodes.additional_networks - region = var.login_nodes.region - static_ips = var.login_nodes.static_ips - subnetwork = var.login_nodes.subnetwork - zone = var.login_nodes.zone - - replace_trigger = var.replace_trigger -} - -resource "google_storage_bucket_object" "startup_scripts" { - for_each = { - for s in var.startup_scripts : format( - "slurm-login-%s-script-%s", local.name, replace(basename(s.filename), "/[^a-zA-Z0-9-_]/", "_") - ) => s.content - } - - bucket = var.slurm_bucket_name - name = "${var.slurm_bucket_dir}/${each.key}" - content = each.value -} - -locals { - name = var.login_nodes.group_name # short hand - - config = { - group_name = local.name - startup_scripts_timeout = var.startup_scripts_timeout - network_storage = var.network_storage - } -} - -resource "google_storage_bucket_object" "config" { - bucket = var.slurm_bucket_name - name = "${var.slurm_bucket_dir}/login_group_configs/${local.name}.yaml" - content = yamlencode(local.config) - - # To ensure that login group "is not ready" until all startup scripts are written down - depends_on = [google_storage_bucket_object.startup_scripts] -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf deleted file mode 100644 index 04de18a188..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "service_account" { - value = module.template.service_account - description = "Service Account used by login VMs" -} - -output "instances" { - value = module.instance.slurm_instances - description = "VM instances of login nodes" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf deleted file mode 100644 index 3efd862942..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/login/variables.tf +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "slurm_cluster_name" { - type = string - description = "Cluster name" -} - -variable "slurm_bucket_path" { - type = string - description = "GCS Bucket URI of Slurm cluster file storage." -} - - -variable "slurm_bucket_name" { - type = string - description = "Name of the bucket for configs" -} - -variable "slurm_bucket_dir" { - type = string - description = "Path to directory in the bucket for configs" -} - - -variable "universe_domain" { - description = "Domain address for alternate API universe" - type = string - default = "googleapis.com" -} - -variable "login_nodes" { - description = "Slurm login instance definitions." - type = object({ - group_name = string - access_config = optional(list(object({ - nat_ip = string - network_tier = string - }))) - additional_disks = optional(list(object({ - disk_name = optional(string) - device_name = optional(string) - disk_size_gb = optional(number) - disk_type = optional(string) - disk_labels = optional(map(string), {}) - auto_delete = optional(bool, true) - boot = optional(bool, false) - disk_resource_manager_tags = optional(map(string), {}) - })), []) - additional_networks = optional(list(object({ - access_config = optional(list(object({ - nat_ip = string - network_tier = string - })), []) - alias_ip_range = optional(list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })), []) - ipv6_access_config = optional(list(object({ - network_tier = string - })), []) - network = optional(string) - network_ip = optional(string, "") - nic_type = optional(string) - queue_count = optional(number) - stack_type = optional(string) - subnetwork = optional(string) - subnetwork_project = optional(string) - })), []) - bandwidth_tier = optional(string, "platform_default") - can_ip_forward = optional(bool, false) - disk_auto_delete = optional(bool, true) - disk_labels = optional(map(string), {}) - disk_resource_manager_tags = optional(map(string), {}) - disk_size_gb = optional(number) - disk_type = optional(string, "n1-standard-1") - enable_confidential_vm = optional(bool, false) - enable_oslogin = optional(bool, true) - enable_shielded_vm = optional(bool, false) - gpu = optional(object({ - count = number - type = string - })) - labels = optional(map(string), {}) - machine_type = optional(string) - advanced_machine_features = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) - metadata = optional(map(string), {}) - min_cpu_platform = optional(string) - num_instances = optional(number, 1) - on_host_maintenance = optional(string) - preemptible = optional(bool, false) - region = optional(string) - resource_manager_tags = optional(map(string), {}) - service_account = optional(object({ - email = optional(string) - scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"]) - })) - shielded_instance_config = optional(object({ - enable_integrity_monitoring = optional(bool, true) - enable_secure_boot = optional(bool, true) - enable_vtpm = optional(bool, true) - })) - source_image_family = optional(string) - source_image_project = optional(string) - source_image = optional(string) - static_ips = optional(list(string), []) - subnetwork = string - spot = optional(bool, false) - tags = optional(list(string), []) - zone = optional(string) - termination_action = optional(string) - }) -} - - -variable "startup_scripts" { - description = "List of scripts to be ran on login VMs startup." - type = list(object({ - filename = string - content = string - })) - default = [] -} - -variable "startup_scripts_timeout" { - description = < - -- [Module: Slurm Nodeset (TPU)](#module-slurm-nodeset-tpu) - - [Overview](#overview) - - [Module API](#module-api) - - - -## Overview - -This is a submodule of [slurm_cluster](../../../slurm_cluster/README.md). It -creates a Slurm TPU nodeset for [slurm_partition](../slurm_partition/README.md). - -## Module API - -For the terraform module API reference, please see -[README_TF.md](./README_TF.md). - - -Copyright (C) SchedMD LLC. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | ~> 1.2 | -| [google](#requirement\_google) | >= 3.53 | -| [null](#requirement\_null) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.53 | -| [null](#provider\_null) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [null_resource.nodeset_tpu](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_subnetwork.nodeset_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [accelerator\_config](#input\_accelerator\_config) | Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details. | 
object({
    topology = string
    version  = string
  })
| 
{
  "topology": "",
  "version": ""
}
| no | -| [data\_disks](#input\_data\_disks) | The data disks to include in the TPU node | `list(string)` | `[]` | no | -| [docker\_image](#input\_docker\_image) | The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf- | `string` | `""` | no | -| [enable\_public\_ip](#input\_enable\_public\_ip) | Enables IP address to access the Internet. | `bool` | `false` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | -| [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of nodes allowed in this partition to be created dynamically. | `number` | `0` | no | -| [node\_count\_static](#input\_node\_count\_static) | Number of nodes to be statically created. | `number` | `0` | no | -| [node\_type](#input\_node\_type) | Specify a node type to base the vm configuration upon it. Not needed if you use accelerator\_config | `string` | `null` | no | -| [nodeset\_name](#input\_nodeset\_name) | Name of Slurm nodeset. | `string` | n/a | yes | -| [preemptible](#input\_preemptible) | Specify whether TPU-vms in this nodeset are preemtible, see https://cloud.google.com/tpu/docs/preemptible for details. | `bool` | `false` | no | -| [preserve\_tpu](#input\_preserve\_tpu) | Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted | `bool` | `true` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [reserved](#input\_reserved) | Specify whether TPU-vms in this nodeset are created under a reservation. | `bool` | `false` | no | -| [service\_account](#input\_service\_account) | Service account to attach to the TPU-vm.
If none is given, the default service account and scopes will be used. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [subnetwork](#input\_subnetwork) | The name of the subnetwork to attach the TPU-vm of this nodeset to. | `string` | n/a | yes | -| [tf\_version](#input\_tf\_version) | Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details. | `string` | n/a | yes | -| [zone](#input\_zone) | Nodes will only be created in this zone. Check https://cloud.google.com/tpu/docs/regions-zones to get zones with TPU-vm in it. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [nodeset](#output\_nodeset) | Nodeset details. | -| [nodeset\_name](#output\_nodeset\_name) | Nodeset name. | -| [service\_account](#output\_service\_account) | Service account object, includes email and scopes. | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf deleted file mode 100644 index 1a6a9cfba1..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/main.tf +++ /dev/null @@ -1,121 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -########### -# NODESET # -########### - -locals { - node_conf_hw = { - Mem334CPU96 = { - CPUs = 96 - Boards = 1 - Sockets = 2 - CoresPerSocket = 24 - ThreadsPerCore = 2 - RealMemory = 307200 - } - Mem400CPU240 = { - CPUs = 240 - Boards = 1 - Sockets = 2 - CoresPerSocket = 60 - ThreadsPerCore = 2 - RealMemory = 400000 - } - } - node_conf_mappings = { - "v2" = local.node_conf_hw.Mem334CPU96 - "v3" = local.node_conf_hw.Mem334CPU96 - "v4" = local.node_conf_hw.Mem400CPU240 - } - simple_nodes = ["v2-8", "v3-8", "v4-8"] -} - -locals { - snetwork = data.google_compute_subnetwork.nodeset_subnetwork.name - region = join("-", slice(split("-", var.zone), 0, 2)) - tpu_fam = var.accelerator_config.version != "" ? lower(var.accelerator_config.version) : split("-", var.node_type)[0] - #If subnetwork is specified and it does not have private_ip_google_access, we need to have public IPs on the TPU - #if no subnetwork is specified, the default one will be used, this does not have private_ip_google_access so we need public IPs too - pub_need = !data.google_compute_subnetwork.nodeset_subnetwork.private_ip_google_access - can_preempt = var.node_type != null ? contains(local.simple_nodes, var.node_type) : false - nodeset_tpu = { - nodeset_name = var.nodeset_name - node_conf = local.node_conf_mappings[local.tpu_fam] - node_type = var.node_type - accelerator_config = var.accelerator_config - tf_version = var.tf_version - preemptible = local.can_preempt ? var.preemptible : false - reserved = var.reserved - node_count_dynamic_max = var.node_count_dynamic_max - node_count_static = var.node_count_static - enable_public_ip = var.enable_public_ip - zone = var.zone - service_account = var.service_account != null ? var.service_account : local.service_account - preserve_tpu = local.can_preempt ? var.preserve_tpu : false - data_disks = var.data_disks - docker_image = var.docker_image != "" ? var.docker_image : "us-docker.pkg.dev/schedmd-slurm-public/tpu/slurm-gcp-6-9:tf-${var.tf_version}" - subnetwork = local.snetwork - network_storage = var.network_storage - } - - service_account = { - email = try(var.service_account.email, null) - scopes = try(var.service_account.scopes, ["https://www.googleapis.com/auth/cloud-platform"]) - } -} - -data "google_compute_subnetwork" "nodeset_subnetwork" { - name = var.subnetwork - region = local.region - project = var.project_id - - self_link = ( - length(regexall("/projects/([^/]*)", var.subnetwork)) > 0 - && length(regexall("/regions/([^/]*)", var.subnetwork)) > 0 - ? var.subnetwork - : null - ) -} - -resource "null_resource" "nodeset_tpu" { - triggers = { - nodeset = sha256(jsonencode(local.nodeset_tpu)) - } - lifecycle { - precondition { - condition = sum([var.node_count_dynamic_max, var.node_count_static]) > 0 - error_message = "Sum of node_count_dynamic_max and node_count_static must be > 0." - } - precondition { - condition = !(var.preemptible && var.reserved) - error_message = "Nodeset cannot be preemptible and reserved at the same time." - } - precondition { - condition = !(var.subnetwork == null && !var.enable_public_ip) - error_message = "Using the default subnetwork for the TPU nodeset requires enable_public_ip set to true." - } - precondition { - condition = !(var.subnetwork != null && (local.pub_need && !var.enable_public_ip)) - error_message = "The subnetwork specified does not have Private Google Access enabled. This is required when enable_public_ip is set to false." - } - precondition { - condition = !(var.node_type == null && (var.accelerator_config.topology == "" && var.accelerator_config.version == "")) - error_message = "Either a node type or an accelerator_config must be provided." - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf deleted file mode 100644 index fce700d567..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "nodeset_name" { - description = "Nodeset name." - value = local.nodeset_tpu.nodeset_name -} - -output "nodeset" { - description = "Nodeset details." - value = local.nodeset_tpu -} - -output "service_account" { - description = "Service account object, includes email and scopes." - value = local.service_account -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf deleted file mode 100644 index a8c470dec9..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/internal/slurm-gcp/nodeset_tpu/variables.tf +++ /dev/null @@ -1,158 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "nodeset_name" { - description = "Name of Slurm nodeset." - type = string - - validation { - condition = can(regex("^[a-z](?:[a-z0-9]{0,14})$", var.nodeset_name)) - error_message = "Variable 'nodeset_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,14})$'." - } -} - -variable "node_type" { - description = "Specify a node type to base the vm configuration upon it. Not needed if you use accelerator_config" - type = string - default = null -} - -variable "accelerator_config" { - description = "Nodeset accelerator config, see https://cloud.google.com/tpu/docs/supported-tpu-configurations for details." - type = object({ - topology = string - version = string - }) - default = { - topology = "" - version = "" - } - validation { - condition = var.accelerator_config.version == "" ? true : contains(["V2", "V3", "V4"], upper(var.accelerator_config.version)) - error_message = "accelerator_config.version must be one of [\"V2\", \"V3\", \"V4\"]" - } - validation { - condition = var.accelerator_config.topology == "" ? true : can(regex("^[1-9]x[1-9](x[1-9])?$", var.accelerator_config.topology)) - error_message = "accelerator_config.topology must be a valid topology, like 2x2 4x4x4 4x2x4 etc..." - } -} - -variable "docker_image" { - description = "The gcp container registry id docker image to use in the TPU vms, it defaults to gcr.io/schedmd-slurm-public/tpu:slurm-gcp-6-9-tf-" - type = string - default = "" -} - -variable "tf_version" { - description = "Nodeset Tensorflow version, see https://cloud.google.com/tpu/docs/supported-tpu-configurations#tpu_vm for details." - type = string -} - -variable "zone" { - description = "Nodes will only be created in this zone. Check https://cloud.google.com/tpu/docs/regions-zones to get zones with TPU-vm in it." - type = string - - validation { - condition = can(coalesce(var.zone)) - error_message = "Zone cannot be null or empty." - } -} - -variable "preemptible" { - description = "Specify whether TPU-vms in this nodeset are preemtible, see https://cloud.google.com/tpu/docs/preemptible for details." - type = bool - default = false -} - -variable "reserved" { - description = "Specify whether TPU-vms in this nodeset are created under a reservation." - type = bool - default = false -} - -variable "preserve_tpu" { - description = "Specify whether TPU-vms will get preserve on suspend, if set to true, on suspend vm is stopped, on false it gets deleted" - type = bool - default = true -} - -variable "node_count_static" { - description = "Number of nodes to be statically created." - type = number - default = 0 - - validation { - condition = var.node_count_static >= 0 - error_message = "Value must be >= 0." - } -} - -variable "node_count_dynamic_max" { - description = "Maximum number of nodes allowed in this partition to be created dynamically." - type = number - default = 0 - - validation { - condition = var.node_count_dynamic_max >= 0 - error_message = "Value must be >= 0." - } -} - -variable "enable_public_ip" { - description = "Enables IP address to access the Internet." - type = bool - default = false -} - -variable "data_disks" { - type = list(string) - description = "The data disks to include in the TPU node" - default = [] -} - -variable "subnetwork" { - description = "The name of the subnetwork to attach the TPU-vm of this nodeset to." - type = string -} - -variable "service_account" { - type = object({ - email = string - scopes = set(string) - }) - description = < -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | > 5.0 | -| [helm](#requirement\_helm) | ~> 2.17 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | > 5.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [install\_gpu\_operator](#module\_install\_gpu\_operator) | ./helm_install | n/a | -| [install\_jobset](#module\_install\_jobset) | ./helm_install | n/a | -| [install\_kueue](#module\_install\_kueue) | ./helm_install | n/a | -| [install\_nvidia\_dra\_driver](#module\_install\_nvidia\_dra\_driver) | ./helm_install | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | -| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [cluster\_id](#input\_cluster\_id) | An identifier for the gke cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | -| [gke\_cluster\_exists](#input\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | `bool` | `false` | no | -| [gpu\_operator](#input\_gpu\_operator) | Install [GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) which uses the [Kubernetes operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) to automate the management of all NVIDIA software components needed to provision GPU. | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | -| [jobset](#input\_jobset) | Install [Jobset](https://github.com/kubernetes-sigs/jobset) which manages a group of K8s [jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) as a unit. | 
object({
    install = optional(bool, false)
    version = optional(string, "v0.7.2")
  })
| `{}` | no | -| [kueue](#input\_kueue) | Install and configure [Kueue](https://kueue.sigs.k8s.io/docs/overview/) workload scheduler. A configuration yaml/template file can be provided with config\_path to be applied right after kueue installation. If a template file provided, its variables can be set to config\_template\_vars. | 
object({
    install              = optional(bool, false)
    version              = optional(string, "v0.11.4")
    config_path          = optional(string, null)
    config_template_vars = optional(map(any), null)
  })
| `{}` | no | -| [nvidia\_dra\_driver](#input\_nvidia\_dra\_driver) | Installs [Nvidia DRA driver](https://github.com/NVIDIA/k8s-dra-driver-gpu) which supports Dynamic Resource Allocation for NVIDIA GPUs in Kubernetes | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0-rc.2")
  })
| `{}` | no | -| [project\_id](#input\_project\_id) | The project ID that hosts the gke cluster. | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md deleted file mode 100644 index 1957899617..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/README.md +++ /dev/null @@ -1,64 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [helm](#requirement\_helm) | ~> 2.17 | - -## Providers - -| Name | Version | -|------|---------| -| [helm](#provider\_helm) | ~> 2.17 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [helm_release.apply_chart](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [atomic](#input\_atomic) | If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used. | `bool` | `false` | no | -| [chart\_name](#input\_chart\_name) | Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL). | `string` | n/a | yes | -| [chart\_repository](#input\_chart\_repository) | URL of the Helm chart repository. Set to null or omit if 'chart\_name' is a path or URL. | `string` | `null` | no | -| [chart\_version](#input\_chart\_version) | Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true). | `string` | `null` | no | -| [cleanup\_on\_fail](#input\_cleanup\_on\_fail) | Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail'). | `bool` | `false` | no | -| [create\_namespace](#input\_create\_namespace) | Set to true to create the namespace if it does not exist ('helm install --create-namespace'). | `bool` | `true` | no | -| [dependency\_update](#input\_dependency\_update) | Run 'helm dependency update' before installing the chart (useful if chart\_name is a local path to an unpacked chart with dependencies). | `bool` | `false` | no | -| [description](#input\_description) | Set an optional description for the Helm release. | `string` | `null` | no | -| [devel](#input\_devel) | Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart\_version' is set, this is ignored. | `bool` | `false` | no | -| [disable\_crd\_hooks](#input\_disable\_crd\_hooks) | Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook'). | `bool` | `false` | no | -| [disable\_openapi\_validation](#input\_disable\_openapi\_validation) | If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation'). | `bool` | `false` | no | -| [disable\_webhooks](#input\_disable\_webhooks) | Prevent hooks from running ('helm install --no-hooks'). | `bool` | `false` | no | -| [force\_update](#input\_force\_update) | Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution. | `bool` | `false` | no | -| [keyring](#input\_keyring) | Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true. | `string` | `null` | no | -| [lint](#input\_lint) | Run the helm chart linter during the plan ('helm lint'). | `bool` | `false` | no | -| [max\_history](#input\_max\_history) | Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit. | `number` | `null` | no | -| [namespace](#input\_namespace) | Kubernetes namespace to install the Helm release into. | `string` | `"default"` | no | -| [pass\_credentials](#input\_pass\_credentials) | Pass credentials to all domains ('helm install --pass-credentials'). Use with caution. | `bool` | `false` | no | -| [postrender](#input\_postrender) | Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary\_path' attribute. | 
object({
    binary_path = string # Path to the post-renderer executable
  })
| `null` | no | -| [recreate\_pods](#input\_recreate\_pods) | Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself. | `bool` | `false` | no | -| [release\_name](#input\_release\_name) | Name of the Helm release. | `string` | n/a | yes | -| [render\_subchart\_notes](#input\_render\_subchart\_notes) | If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes'). | `bool` | `false` | no | -| [reset\_values](#input\_reset\_values) | When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values'). | `bool` | `false` | no | -| [reuse\_values](#input\_reuse\_values) | When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset\_values' is specified, this is ignored. | `bool` | `false` | no | -| [set\_values](#input\_set\_values) | List of objects defining values to set ('helm install --set'). | 
list(object({
    name  = string                     # Path to the value (e.g., 'service.type', 'replicaCount')
    value = string                     # The value to set
    type  = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file')
  }))
| `[]` | no | -| [skip\_crds](#input\_skip\_crds) | If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present. | `bool` | `false` | no | -| [timeout](#input\_timeout) | Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout'). | `number` | `300` | no | -| [values\_yaml](#input\_values\_yaml) | List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile(). | `list(string)` | `[]` | no | -| [verify](#input\_verify) | Verify the package before installing it ('helm install --verify'). | `bool` | `false` | no | -| [wait](#input\_wait) | Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait'). | `bool` | `true` | no | -| [wait\_for\_jobs](#input\_wait\_for\_jobs) | If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs'). | `bool` | `false` | no | - -## Outputs - -No outputs. - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf deleted file mode 100644 index bd2383b772..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/main.tf +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resource "helm_release" "apply_chart" { - # Required Identification - name = var.release_name - chart = var.chart_name - - # Chart Source & Version - repository = var.chart_repository - version = var.chart_version - devel = var.devel - - # Target Namespace - namespace = var.namespace - create_namespace = var.create_namespace - - # Values Configuration - values = var.values_yaml - - dynamic "set" { - for_each = var.set_values - content { - name = set.value.name - value = set.value.value - type = set.value.type - } - } - - # Installation/Upgrade Behavior - description = var.description - atomic = var.atomic - cleanup_on_fail = var.cleanup_on_fail - dependency_update = var.dependency_update - disable_crd_hooks = var.disable_crd_hooks - disable_openapi_validation = var.disable_openapi_validation - disable_webhooks = var.disable_webhooks - force_update = var.force_update - lint = var.lint - max_history = var.max_history - recreate_pods = var.recreate_pods # Note: Deprecated in Helm CLI - render_subchart_notes = var.render_subchart_notes - reset_values = var.reset_values - reuse_values = var.reuse_values - skip_crds = var.skip_crds - timeout = var.timeout - wait = var.wait - wait_for_jobs = var.wait_for_jobs - - # Verification & Credentials - keyring = var.keyring - pass_credentials = var.pass_credentials - verify = var.verify - - # Post Rendering - dynamic "postrender" { - # Only include the block if var.postrender is not null - for_each = var.postrender == null ? [] : [var.postrender] - content { - binary_path = postrender.value.binary_path - } - } - -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml deleted file mode 100644 index e18197e2b7..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf deleted file mode 100644 index 04e8e214fc..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/variables.tf +++ /dev/null @@ -1,212 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Description: Input variables for the generic Helm release module. - -# --- Required --- -variable "release_name" { - description = "Name of the Helm release." - type = string -} - -variable "chart_name" { - description = "Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL)." - type = string -} - -# --- Chart Location & Version --- -variable "chart_repository" { - description = "URL of the Helm chart repository. Set to null or omit if 'chart_name' is a path or URL." - type = string - default = null -} - -variable "chart_version" { - description = "Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true)." - type = string - default = null -} - -variable "devel" { - description = "Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart_version' is set, this is ignored." - type = bool - default = false -} - -# --- Namespace --- -variable "namespace" { - description = "Kubernetes namespace to install the Helm release into." - type = string - default = "default" -} - -variable "create_namespace" { - description = "Set to true to create the namespace if it does not exist ('helm install --create-namespace')." - type = bool - default = true # Common convenience setting -} - -# --- Values Customization --- -variable "values_yaml" { - description = "List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile()." - type = list(string) - default = [] -} - -variable "set_values" { - description = "List of objects defining values to set ('helm install --set')." - type = list(object({ - name = string # Path to the value (e.g., 'service.type', 'replicaCount') - value = string # The value to set - type = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file') - })) - default = [] -} - -# --- Installation/Upgrade Behavior --- -variable "description" { - description = "Set an optional description for the Helm release." - type = string - default = null -} - -variable "atomic" { - description = "If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used." - type = bool - default = false -} - -variable "wait" { - description = "Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait')." - type = bool - default = true # Often a good default for dependencies -} - -variable "wait_for_jobs" { - description = "If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs')." - type = bool - default = false # Helm CLI default is false -} - -variable "timeout" { - description = "Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout')." - type = number - default = 300 # 5 minutes (Helm CLI default) -} - -variable "cleanup_on_fail" { - description = "Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail')." - type = bool - default = false -} - -variable "dependency_update" { - description = "Run 'helm dependency update' before installing the chart (useful if chart_name is a local path to an unpacked chart with dependencies)." - type = bool - default = false -} - -variable "disable_crd_hooks" { - description = "Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook')." - type = bool - default = false -} - -variable "disable_openapi_validation" { - description = "If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation')." - type = bool - default = false -} - -variable "disable_webhooks" { - description = "Prevent hooks from running ('helm install --no-hooks')." - type = bool - default = false -} - -variable "force_update" { - description = "Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution." - type = bool - default = false -} - -variable "lint" { - description = "Run the helm chart linter during the plan ('helm lint')." - type = bool - default = false -} - -variable "max_history" { - description = "Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit." - type = number - default = null # Terraform provider defaults to Helm's default (usually 10) -} - -variable "recreate_pods" { - description = "Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself." - type = bool - default = false -} - -variable "render_subchart_notes" { - description = "If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes')." - type = bool - default = false -} - -variable "reset_values" { - description = "When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values')." - type = bool - default = false -} - -variable "reuse_values" { - description = "When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset_values' is specified, this is ignored." - type = bool - default = false # Helm CLI default is false -} - -variable "skip_crds" { - description = "If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present." - type = bool - default = false -} - -# --- Verification & Credentials --- -variable "keyring" { - description = "Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true." - type = string - default = null # Defaults to Helm's default keyring location -} - -variable "pass_credentials" { - description = "Pass credentials to all domains ('helm install --pass-credentials'). Use with caution." - type = bool - default = false -} - -variable "verify" { - description = "Verify the package before installing it ('helm install --verify')." - type = bool - default = false -} - -# --- Advanced Rendering --- -variable "postrender" { - description = "Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary_path' attribute." - type = object({ - binary_path = string # Path to the post-renderer executable - }) - default = null # Disabled by default -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf deleted file mode 100644 index 09d912e2c9..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/helm_install/versions.tf +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_providers { - helm = { - source = "hashicorp/helm" - version = "~> 2.17" - } - } - - required_version = ">= 1.3" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md deleted file mode 100644 index 46bfe51a32..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/README.md +++ /dev/null @@ -1,40 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [kubernetes](#requirement\_kubernetes) | ~> 2.23 | - -## Providers - -| Name | Version | -|------|---------| -| [kubernetes](#provider\_kubernetes) | ~> 2.23 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [kubernetes_manifest.apply_manifests](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [content](#input\_content) | The YAML body to apply to gke cluster. | `string` | `null` | no | -| [field\_manager](#input\_field\_manager) | (Optional) Configure field manager options. The `name` is the name of the field manager. The `force_conflicts` flag allows overriding conflicts. | 
object({
    name            = optional(string, null)
    force_conflicts = optional(bool, false)
  })
| `null` | no | -| [resource\_timeouts](#input\_resource\_timeouts) | (Optional) Configure custom timeouts for the create, update, and delete operations of the resource. These timeouts also govern the duration for any 'wait' conditions to be met. | 
object({
    create = optional(string, null)
    update = optional(string, null)
    delete = optional(string, null)
  })
| 
{
  "create": "15m",
  "delete": "5m",
  "update": "10m"
}
| no | -| [source\_path](#input\_source\_path) | The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file. | `string` | `""` | no | -| [template\_vars](#input\_template\_vars) | The values to populate template file(s) with. | `any` | `null` | no | -| [wait\_for\_fields](#input\_wait\_for\_fields) | (Optional) A map of attribute paths and desired patterns to be matched. After each apply the provider will wait for all attributes listed here to reach a value that matches the desired pattern. | `map(string)` | `{}` | no | -| [wait\_for\_rollout](#input\_wait\_for\_rollout) | Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details. | `bool` | `true` | no | - -## Outputs - -No outputs. - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf deleted file mode 100644 index f97f26038d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/main.tf +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - yaml_separator = "\n---" - - # --- 1. Determine the primary source of YAML content --- - # Prioritize 'content' variable if provided - primary_content_body = var.content != "" ? var.content : null - - # --- 2. Handle 'source_path' based on its type (File vs. Directory) --- - - # Check if source_path is a directory (indicated by trailing slash) - is_directory = endswith(var.source_path, "/") - directory_absolute_path = local.is_directory ? abspath(var.source_path) : null - - # Check if source_path is a single yaml or tftpl file (only if not a directory) - is_single_file = !local.is_directory && ( - length(regexall("\\.yaml$", lower(var.source_path))) > 0 || - length(regexall("\\.tftpl$", lower(var.source_path))) > 0 - ) - single_file_raw_content = local.is_single_file ? ( - length(regexall("\\.tftpl$", lower(var.source_path))) > 0 ? - templatefile(abspath(var.source_path), var.template_vars) : - file(abspath(var.source_path)) - ) : null - - # Docs from primary_content_body - docs_from_primary_source = [ - for doc in split(local.yaml_separator, coalesce(local.primary_content_body, local.single_file_raw_content, "")) : trimspace(doc) - if length(trimspace(doc)) > 0 - ] - - # Docs from .yaml files in a directory - directory_yaml_files = local.is_directory ? fileset(local.directory_absolute_path, "*.yaml") : [] - docs_from_directory_yamls = flatten([ - for file_name in local.directory_yaml_files : - [ - for doc in split(local.yaml_separator, file(format("%s/%s", local.directory_absolute_path, file_name))) : trimspace(doc) - if length(trimspace(doc)) > 0 - ] - ]) - - # Docs from .tftpl files in a directory - directory_template_files = local.is_directory ? fileset(local.directory_absolute_path, "*.tftpl") : [] - docs_from_directory_templates = flatten([ - for file_name in local.directory_template_files : - [ - for doc in split(local.yaml_separator, templatefile(format("%s/%s", local.directory_absolute_path, file_name), var.template_vars)) : trimspace(doc) - if length(trimspace(doc)) > 0 - ] - ]) - - all_parsed_docs = concat( - local.docs_from_primary_source, - local.docs_from_directory_yamls, - local.docs_from_directory_templates - ) - - # --- 5. Create the final map for `for_each` (keys must be unique strings) --- - docs_map = tomap({ - for index, doc in local.all_parsed_docs : index => doc - if length(trimspace(doc)) > 0 - }) -} - -# Apply all manifest files dynamically -resource "kubernetes_manifest" "apply_manifests" { - for_each = local.docs_map - manifest = yamldecode(each.value) - timeouts { - create = var.resource_timeouts.create - update = var.resource_timeouts.update - delete = var.resource_timeouts.delete - } - - dynamic "wait" { - for_each = var.wait_for_rollout ? [1] : [] - content { - rollout = var.wait_for_rollout - fields = var.wait_for_fields - } - } - - # Configure the 'field_manager' block dynamically - dynamic "field_manager" { - for_each = var.field_manager != null ? [var.field_manager] : [] - content { - name = field_manager.value.name - force_conflicts = field_manager.value.force_conflicts - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml deleted file mode 100644 index e18197e2b7..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf deleted file mode 100644 index 0b846189ea..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/variables.tf +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Description: Input variables for the generic Helm release module. - -variable "content" { - description = "The YAML body to apply to gke cluster." - type = string - default = null -} - -variable "source_path" { - description = "The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file." - type = string - default = "" -} - -variable "template_vars" { - description = "The values to populate template file(s) with." - type = any - default = null -} - -variable "wait_for_rollout" { - description = "Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details." - type = bool - default = true -} - - -variable "wait_for_fields" { - description = "(Optional) A map of attribute paths and desired patterns to be matched. After each apply the provider will wait for all attributes listed here to reach a value that matches the desired pattern." - type = map(string) - default = {} -} - -variable "resource_timeouts" { - description = "(Optional) Configure custom timeouts for the create, update, and delete operations of the resource. These timeouts also govern the duration for any 'wait' conditions to be met." - type = object({ - create = optional(string, null) - update = optional(string, null) - delete = optional(string, null) - }) - default = { - create = "15m" # Default create timeout, also covers waiting for initial conditions - update = "10m" # Default update timeout, also covers waiting for update conditions - delete = "5m" # Default delete timeout - } -} - -variable "field_manager" { - description = "(Optional) Configure field manager options. The `name` is the name of the field manager. The `force_conflicts` flag allows overriding conflicts." - type = object({ - name = optional(string, null) - force_conflicts = optional(bool, false) - }) - default = null -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf deleted file mode 100644 index 61786b06de..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/kubernetes_manifest/versions.tf +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - # Defines the providers that this module depends on and their versions. - required_providers { - kubernetes = { - source = "hashicorp/kubernetes" - version = "~> 2.23" - } - } - required_version = ">= 1.3" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/main.tf deleted file mode 100644 index 8db4870452..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/management/dependencies-installer/main.tf +++ /dev/null @@ -1,183 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - cluster_id_parts = split("/", var.cluster_id) - cluster_name = local.cluster_id_parts[5] - cluster_location = local.cluster_id_parts[3] - project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] - - install_gpu_operator = try(var.gpu_operator.install, false) - install_nvidia_dra_driver = try(var.nvidia_dra_driver.install, false) -} - -data "google_container_cluster" "gke_cluster" { - project = local.project_id - name = local.cluster_name - location = local.cluster_location -} - -data "google_client_config" "default" {} - -module "install_kueue" { - source = "./helm_install" - depends_on = [var.gke_cluster_exists] - - release_name = "kueue" - - chart_name = "oci://registry.k8s.io/kueue/charts/kueue" - chart_version = var.kueue.version # Specify your desired Kueue version - - create_namespace = true # Helm can also create the namespace - wait = true - timeout = 600 # seconds -} - -module "install_jobset" { - source = "./helm_install" - depends_on = [var.gke_cluster_exists, module.install_kueue] - release_name = "jobset-controller" # The release name for your JobSet installation - chart_name = "oci://registry.k8s.io/jobset/charts/jobset" # The Helm repository URL for nvidia charts - chart_version = var.jobset.version - create_namespace = true - namespace = "jobset-system" -} - -module "install_nvidia_dra_driver" { - count = local.install_nvidia_dra_driver ? 1 : 0 - depends_on = [var.gke_cluster_exists] - source = "./helm_install" - - release_name = "nvidia-dra-driver-gpu" # The release name - chart_repository = "https://helm.ngc.nvidia.com/nvidia" # The Helm repository URL for nvidia charts - chart_name = "nvidia-dra-driver-gpu" # The chart name - chart_version = var.nvidia_dra_driver.version # The chart version - namespace = "nvidia-dra-driver-gpu" # The target namespace - create_namespace = true # Equivalent to --create-namespace - - # Use the 'values' argument to pass the YAML content - # This corresponds to the -f <(cat < -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.2 | -| [google](#requirement\_google) | >= 3.83 | -| [google-beta](#requirement\_google-beta) | >= 3.83 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [google-beta](#provider\_google-beta) | >= 3.83 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_global_address.private_ip_alloc](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_global_address) | resource | -| [google_service_networking_connection.private_vpc_connection](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_networking_connection) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [address](#input\_address) | The IP address or beginning of the address range allocated for the private service access. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to supporting resources. Key-value pairs. | `map(string)` | n/a | yes | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to configure private service Access.:
`projects//global/networks/`" | `string` | n/a | yes | -| [prefix\_length](#input\_prefix\_length) | The prefix length of the IP range allocated for the private service access. | `number` | `16` | no | -| [project\_id](#input\_project\_id) | ID of project in which Private Service Access will be created. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [cidr\_range](#output\_cidr\_range) | CIDR range of the created google\_compute\_global\_address | -| [connect\_mode](#output\_connect\_mode) | Services that use Private Service Access typically specify connect\_mode
"PRIVATE\_SERVICE\_ACCESS". This output value sets connect\_mode and additionally
blocks terraform actions until the VPC connection has been created. | -| [private\_vpc\_connection\_peering](#output\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection that was created by the service provider. | -| [reserved\_ip\_range](#output\_reserved\_ip\_range) | Named IP range to be used by services connected with Private Service Access. | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/main.tf deleted file mode 100644 index 4bb54821c3..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/main.tf +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "private-service-access", ghpc_role = "network" }) -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_compute_global_address" "private_ip_alloc" { - provider = google-beta - name = "global-psconnect-ip-${random_id.resource_name_suffix.hex}" - project = var.project_id - purpose = "VPC_PEERING" - address_type = "INTERNAL" - network = var.network_id - prefix_length = var.prefix_length - labels = local.labels - address = var.address -} - -resource "google_service_networking_connection" "private_vpc_connection" { - network = var.network_id - service = "servicenetworking.googleapis.com" - reserved_peering_ranges = [google_compute_global_address.private_ip_alloc.name] -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/metadata.yaml deleted file mode 100644 index 93e8b3970e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - servicenetworking.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/outputs.tf deleted file mode 100644 index 296f2e9140..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/outputs.tf +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "private_vpc_connection_peering" { - description = "The name of the VPC Network peering connection that was created by the service provider." - sensitive = true - value = google_service_networking_connection.private_vpc_connection.peering -} - -output "connect_mode" { - description = <<-EOT - Services that use Private Service Access typically specify connect_mode - "PRIVATE_SERVICE_ACCESS". This output value sets connect_mode and additionally - blocks terraform actions until the VPC connection has been created. - EOT - value = "PRIVATE_SERVICE_ACCESS" - depends_on = [ - google_service_networking_connection.private_vpc_connection, - ] -} - -output "reserved_ip_range" { - description = "Named IP range to be used by services connected with Private Service Access." - value = google_compute_global_address.private_ip_alloc.name -} - -output "cidr_range" { - description = "CIDR range of the created google_compute_global_address" - value = "${google_compute_global_address.private_ip_alloc.address}/${google_compute_global_address.private_ip_alloc.prefix_length}" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/variables.tf deleted file mode 100644 index 18b73ac2d9..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/variables.tf +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "address" { - description = "The IP address or beginning of the address range allocated for the private service access." - type = string - default = null -} - -variable "network_id" { - description = <<-EOT - The ID of the GCE VPC network to configure private service Access.: - `projects//global/networks/`" - EOT - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "labels" { - description = "Labels to add to supporting resources. Key-value pairs." - type = map(string) -} - -variable "prefix_length" { - description = "The prefix length of the IP range allocated for the private service access." - type = number - default = 16 -} - -variable "project_id" { - description = "ID of project in which Private Service Access will be created." - type = string -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/versions.tf deleted file mode 100644 index c5a889017f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/network/private-service-access/versions.tf +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - google-beta = { - source = "hashicorp/google-beta" - version = ">= 3.83" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.57.0" - } - - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:private-service-access/v1.57.0" - } - - required_version = ">= 1.2" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/new-project/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/project/new-project/README.md deleted file mode 100644 index 5e5cabe9d5..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/project/new-project/README.md +++ /dev/null @@ -1,128 +0,0 @@ -## Description - -This module allows you to create opinionated Google Cloud Platform projects. It -creates projects and configures aspects like Shared VPC connectivity, IAM -access, Service Accounts, and API enablement to follow best practices. - -This module is meant for use with Terraform 0.13. - -**Note:** This module has been removed from the Cluster Toolkit. The upstream module (`terraform-google-project-factory`) is now the recommended way to create and manage GCP projects. - -### Example - -```yaml -- id: project - source: github.com/terraform-google-modules/terraform-google-project-factory?rev=v17.0.0&depth=1 -``` - -This creates a new project with pre-defined project ID, a designated folder and -organization and associated billing account which will be used to pay for -services consumed. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [project\_factory](#module\_project\_factory) | terraform-google-modules/project-factory/google | ~> 11.3 | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [activate\_api\_identities](#input\_activate\_api\_identities) | The list of service identities (Google Managed service account for the API) to force-create for the project (e.g. in order to grant additional roles).
APIs in this list will automatically be appended to `activate_apis`.
Not including the API in this list will follow the default behaviour for identity creation (which is usually when the first resource using the API is created).
Any roles (e.g. service agent role) must be explicitly listed. See https://cloud.google.com/iam/docs/understanding-roles#service-agent-roles-roles for a list of related roles. | 
list(object({
    api   = string
    roles = list(string)
  }))
| `[]` | no | -| [activate\_apis](#input\_activate\_apis) | The list of apis to activate within the project | `list(string)` | 
[
  "compute.googleapis.com",
  "serviceusage.googleapis.com",
  "storage.googleapis.com"
]
| no | -| [auto\_create\_network](#input\_auto\_create\_network) | Create the default network | `bool` | `false` | no | -| [billing\_account](#input\_billing\_account) | The ID of the billing account to associate this project with | `string` | n/a | yes | -| [bucket\_force\_destroy](#input\_bucket\_force\_destroy) | Force the deletion of all objects within the GCS bucket when deleting the bucket (optional) | `bool` | `false` | no | -| [bucket\_labels](#input\_bucket\_labels) | A map of key/value label pairs to assign to the bucket (optional) | `map(string)` | `{}` | no | -| [bucket\_location](#input\_bucket\_location) | The location for a GCS bucket to create (optional) | `string` | `"US"` | no | -| [bucket\_name](#input\_bucket\_name) | A name for a GCS bucket to create (in the bucket\_project project), useful for Terraform state (optional) | `string` | `""` | no | -| [bucket\_project](#input\_bucket\_project) | A project to create a GCS bucket (bucket\_name) in, useful for Terraform state (optional) | `string` | `""` | no | -| [bucket\_ula](#input\_bucket\_ula) | Enable Uniform Bucket Level Access | `bool` | `true` | no | -| [bucket\_versioning](#input\_bucket\_versioning) | Enable versioning for a GCS bucket to create (optional) | `bool` | `false` | no | -| [budget\_alert\_pubsub\_topic](#input\_budget\_alert\_pubsub\_topic) | The name of the Cloud Pub/Sub topic where budget related messages will be published, in the form of `projects/{project_id}/topics/{topic_id}` | `string` | `null` | no | -| [budget\_alert\_spent\_percents](#input\_budget\_alert\_spent\_percents) | A list of percentages of the budget to alert on when threshold is exceeded | `list(number)` | 
[
  0.5,
  0.7,
  1
]
| no | -| [budget\_amount](#input\_budget\_amount) | The amount to use for a budget alert | `number` | `null` | no | -| [budget\_display\_name](#input\_budget\_display\_name) | The display name of the budget. If not set defaults to `Budget For ` | `string` | `null` | no | -| [budget\_monitoring\_notification\_channels](#input\_budget\_monitoring\_notification\_channels) | A list of monitoring notification channels in the form `[projects/{project_id}/notificationChannels/{channel_id}]`. A maximum of 5 channels are allowed. | `list(string)` | `[]` | no | -| [consumer\_quotas](#input\_consumer\_quotas) | The quotas configuration you want to override for the project. | 
list(object({
    service = string,
    metric  = string,
    limit   = string,
    value   = string,
  }))
| `[]` | no | -| [create\_project\_sa](#input\_create\_project\_sa) | Whether the default service account for the project shall be created | `bool` | `true` | no | -| [default\_network\_tier](#input\_default\_network\_tier) | Default Network Service Tier for resources created in this project. If unset, the value will not be modified. See https://cloud.google.com/network-tiers/docs/using-network-service-tiers and https://cloud.google.com/network-tiers. | `string` | `""` | no | -| [default\_service\_account](#input\_default\_service\_account) | Project default service account setting: can be one of `delete`, `deprivilege`, `disable`, or `keep`. | `string` | `"keep"` | no | -| [disable\_dependent\_services](#input\_disable\_dependent\_services) | Whether services that are enabled and which depend on this service should also be disabled when this service is destroyed. | `bool` | `true` | no | -| [disable\_services\_on\_destroy](#input\_disable\_services\_on\_destroy) | Whether project services will be disabled when the resources are destroyed | `bool` | `true` | no | -| [domain](#input\_domain) | The domain name (optional). | `string` | `""` | no | -| [enable\_shared\_vpc\_host\_project](#input\_enable\_shared\_vpc\_host\_project) | If this project is a shared VPC host project. If true, you must *not* set svpc\_host\_project\_id variable. Default is false. | `bool` | `false` | no | -| [folder\_id](#input\_folder\_id) | The ID of a folder to host this project | `string` | `""` | no | -| [grant\_services\_network\_role](#input\_grant\_services\_network\_role) | Whether or not to grant service agents the network roles on the host project | `bool` | `true` | no | -| [grant\_services\_security\_admin\_role](#input\_grant\_services\_security\_admin\_role) | Whether or not to grant Kubernetes Engine Service Agent the Security Admin role on the host project so it can manage firewall rules | `bool` | `false` | no | -| [group\_name](#input\_group\_name) | A group to control the project by being assigned group\_role (defaults to project editor) | `string` | `""` | no | -| [group\_role](#input\_group\_role) | The role to give the controlling group (group\_name) over the project (defaults to project editor) | `string` | `"roles/editor"` | no | -| [labels](#input\_labels) | Map of labels for project | `map(string)` | `{}` | no | -| [lien](#input\_lien) | Add a lien on the project to prevent accidental deletion | `bool` | `false` | no | -| [name](#input\_name) | The name for the project | `string` | `null` | no | -| [org\_id](#input\_org\_id) | The organization ID. | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | The ID to give the project. If not provided, the `name` will be used. | `string` | `""` | no | -| [project\_sa\_name](#input\_project\_sa\_name) | Default service account name for the project. | `string` | `"project-service-account"` | no | -| [random\_project\_id](#input\_random\_project\_id) | Adds a suffix of 4 random characters to the `project_id` | `bool` | `false` | no | -| [sa\_role](#input\_sa\_role) | A role to give the default Service Account for the project (defaults to none) | `string` | `""` | no | -| [shared\_vpc\_subnets](#input\_shared\_vpc\_subnets) | List of subnets fully qualified subnet IDs (ie. projects/$project\_id/regions/$region/subnetworks/$subnet\_id) | `list(string)` | `[]` | no | -| [svpc\_host\_project\_id](#input\_svpc\_host\_project\_id) | The ID of the host project which hosts the shared VPC | `string` | `""` | no | -| [usage\_bucket\_name](#input\_usage\_bucket\_name) | Name of a GCS bucket to store GCE usage reports in (optional) | `string` | `""` | no | -| [usage\_bucket\_prefix](#input\_usage\_bucket\_prefix) | Prefix in the GCS bucket to store GCE usage reports in (optional) | `string` | `""` | no | -| [vpc\_service\_control\_attach\_enabled](#input\_vpc\_service\_control\_attach\_enabled) | Whether the project will be attached to a VPC Service Control Perimeter | `bool` | `false` | no | -| [vpc\_service\_control\_perimeter\_name](#input\_vpc\_service\_control\_perimeter\_name) | The name of a VPC Service Control Perimeter to add the created project to | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [api\_s\_account](#output\_api\_s\_account) | API service account email | -| [api\_s\_account\_fmt](#output\_api\_s\_account\_fmt) | API service account email formatted for terraform use | -| [budget\_name](#output\_budget\_name) | The name of the budget if created | -| [domain](#output\_domain) | The organization's domain | -| [enabled\_api\_identities](#output\_enabled\_api\_identities) | Enabled API identities in the project | -| [enabled\_apis](#output\_enabled\_apis) | Enabled APIs in the project | -| [group\_email](#output\_group\_email) | The email of the G Suite group with group\_name | -| [project\_bucket\_self\_link](#output\_project\_bucket\_self\_link) | Project's bucket selfLink | -| [project\_bucket\_url](#output\_project\_bucket\_url) | Project's bucket url | -| [project\_id](#output\_project\_id) | ID of the project that was created | -| [project\_name](#output\_project\_name) | Name of the project that was created | -| [project\_number](#output\_project\_number) | Number of the project that was created | -| [service\_account\_display\_name](#output\_service\_account\_display\_name) | The display name of the default service account | -| [service\_account\_email](#output\_service\_account\_email) | The email of the default service account | -| [service\_account\_id](#output\_service\_account\_id) | The id of the default service account | -| [service\_account\_name](#output\_service\_account\_name) | The fully-qualified name of the default service account | -| [service\_account\_unique\_id](#output\_service\_account\_unique\_id) | The unique id of the default service account | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/README.md deleted file mode 100644 index 0f5c10c7e4..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/README.md +++ /dev/null @@ -1,111 +0,0 @@ -## Description - -Allows creation of service accounts for a Google Cloud Platform project. - -### Example - -```yaml -- id: service_acct - source: community/modules/project/service-account - settings: - project_id: $(vars.project_id) - name: instance_acct - project_roles: - - logging.logWriter - - monitoring.metricWriter - - storage.objectViewer -``` - -This creates a service account in GCP project "project_id" with the name -"instance_acct". It will have the 3 roles listed for all resources within the -project. - -### Usage with startup-script module - -When this module is used in conjunction with the [startup-script] module, the -service account must be granted (at least) read access to the bucket. This can -be achieved by granting project-wide access as shown above or by specifying the -service account as a bucket viewer in the startup-script module: - -```yaml -- id: service_acct - source: community/modules/project/service-account - settings: - project_id: $(vars.project_id) - name: instance_acct - project_roles: - - logging.logWriter - - monitoring.metricWriter -- id: script - source: modules/scripts/startup-script - settings: - bucket_viewers: - - $(service_acct.service_account_iam_email) -``` - -[startup-script]: ../../../../modules/scripts/startup-script/README.md - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [service\_account](#module\_service\_account) | terraform-google-modules/service-accounts/google | ~> 4.2 | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [billing\_account\_id](#input\_billing\_account\_id) | If assigning billing role, specify a billing account (default is to assign at the organizational level). | `string` | `""` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment (will be prepended to service account name) | `string` | n/a | yes | -| [description](#input\_description) | Description of the created service account. | `string` | `"Service Account"` | no | -| [descriptions](#input\_descriptions) | Deprecated; create single service accounts using var.description. | `list(string)` | `null` | no | -| [display\_name](#input\_display\_name) | Display name of the created service account. | `string` | `"Service Account"` | no | -| [generate\_keys](#input\_generate\_keys) | Generate keys for service account. | `bool` | `false` | no | -| [grant\_billing\_role](#input\_grant\_billing\_role) | Grant billing user role. | `bool` | `false` | no | -| [grant\_xpn\_roles](#input\_grant\_xpn\_roles) | Grant roles for shared VPC management. | `bool` | `true` | no | -| [name](#input\_name) | Name of the service account to create. | `string` | n/a | yes | -| [names](#input\_names) | Deprecated; create single service accounts using var.name. | `list(string)` | `null` | no | -| [org\_id](#input\_org\_id) | Id of the organization for org-level roles. | `string` | `""` | no | -| [prefix](#input\_prefix) | Deprecated; prefix now set using var.deployment\_name | `string` | `null` | no | -| [project\_id](#input\_project\_id) | ID of the project | `string` | n/a | yes | -| [project\_roles](#input\_project\_roles) | List of roles to grant to service account (e.g. "storage.objectViewer" or "compute.instanceAdmin.v1" | `list(string)` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [key](#output\_key) | Service account key (if creation was requested) | -| [service\_account\_email](#output\_service\_account\_email) | Service account e-mail address | -| [service\_account\_iam\_email](#output\_service\_account\_iam\_email) | Service account IAM binding format (serviceAccount:name@example.com) | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/main.tf deleted file mode 100644 index e8a69be642..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/main.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - display_name = "${var.display_name} (${var.deployment_name})" - description = "${var.description} (${var.deployment_name})" -} - -module "service_account" { - source = "terraform-google-modules/service-accounts/google" - version = "~> 4.2" - - billing_account_id = var.billing_account_id - description = local.description - display_name = local.display_name - generate_keys = var.generate_keys - grant_billing_role = var.grant_billing_role - grant_xpn_roles = var.grant_xpn_roles - names = [var.name] - org_id = var.org_id - prefix = var.deployment_name - project_id = var.project_id - project_roles = [for role in var.project_roles : "${var.project_id}=>roles/${role}"] -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/metadata.yaml deleted file mode 100644 index c4dcdffdf4..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - iam.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/outputs.tf deleted file mode 100644 index f9c9be05c8..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/outputs.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "key" { - description = "Service account key (if creation was requested)" - value = module.service_account.key -} - -output "service_account_email" { - description = "Service account e-mail address" - value = module.service_account.email - depends_on = [ - module.service_account, - ] -} - -output "service_account_iam_email" { - description = "Service account IAM binding format (serviceAccount:name@example.com)" - value = module.service_account.iam_email - depends_on = [ - module.service_account, - ] -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/variables.tf deleted file mode 100644 index 53267f47e7..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/variables.tf +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "billing_account_id" { - description = "If assigning billing role, specify a billing account (default is to assign at the organizational level)." - type = string - default = "" -} - -variable "deployment_name" { - description = "Name of the deployment (will be prepended to service account name)" - type = string -} - -variable "description" { - description = "Description of the created service account." - type = string - default = "Service Account" -} - -# tflint-ignore: terraform_unused_declarations -variable "descriptions" { - description = "Deprecated; create single service accounts using var.description." - type = list(string) - default = null - - validation { - condition = var.descriptions == null - error_message = "var.descriptions has been deprecated in favor of creating single accounts with var.description" - } -} - -variable "display_name" { - description = "Display name of the created service account." - type = string - default = "Service Account" -} - -variable "generate_keys" { - description = "Generate keys for service account." - type = bool - default = false -} - -variable "grant_billing_role" { - description = "Grant billing user role." - type = bool - default = false -} - -variable "grant_xpn_roles" { - description = "Grant roles for shared VPC management." - type = bool - default = true -} - -variable "name" { - description = "Name of the service account to create." - type = string -} - -# tflint-ignore: terraform_unused_declarations -variable "names" { - description = "Deprecated; create single service accounts using var.name." - type = list(string) - default = null - - validation { - condition = var.names == null - error_message = "var.names has been deprecated in favor of creating single accounts with var.name" - } -} - -variable "org_id" { - description = "Id of the organization for org-level roles." - type = string - default = "" -} - -# tflint-ignore: terraform_unused_declarations -variable "prefix" { - description = "Deprecated; prefix now set using var.deployment_name" - type = string - default = null - - validation { - condition = var.prefix == null - error_message = "var.prefix has been deprecated in favor of setting prefix with var.deployment_name" - } -} - -variable "project_id" { - description = "ID of the project" - type = string -} - -variable "project_roles" { - description = "List of roles to grant to service account (e.g. \"storage.objectViewer\" or \"compute.instanceAdmin.v1\"" - type = list(string) -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/versions.tf deleted file mode 100644 index 38e6e71945..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-account/versions.tf +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - } - - required_version = ">= 0.14.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/README.md deleted file mode 100644 index 266eac26ec..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/README.md +++ /dev/null @@ -1,70 +0,0 @@ -## Description - -Allows management of multiple API services for a Google Cloud Platform project. - -### Example - -```yaml -- id: services-api - source: community/modules/project/service-enablement - settings: - gcp_service_list: [ - "file.googleapis.com", - "compute.googleapis.com" - ] -``` - -This allows the project to enable both the filestore API as well as the compute API. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_project_service.gcp_services](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_service) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [disable\_on\_destroy](#input\_disable\_on\_destroy) | Disable services on destroy if they were enabled (or already enabled) during apply (default: false) | `bool` | `false` | no | -| [gcp\_service\_list](#input\_gcp\_service\_list) | list of APIs to be enabled for the project | `list(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | ID of the project | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/main.tf deleted file mode 100644 index 965e93c549..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/main.tf +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -resource "google_project_service" "gcp_services" { - count = length(var.gcp_service_list) - project = var.project_id - service = var.gcp_service_list[count.index] - timeouts { - create = "30m" - update = "40m" - } - - disable_dependent_services = true - disable_on_destroy = var.disable_on_destroy -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/metadata.yaml deleted file mode 100644 index c594c8f819..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - serviceusage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/variables.tf deleted file mode 100644 index 08f13999fe..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/variables.tf +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "ID of the project" - type = string -} - -variable "gcp_service_list" { - description = "list of APIs to be enabled for the project" - type = list(string) -} - -variable "disable_on_destroy" { - description = "Disable services on destroy if they were enabled (or already enabled) during apply (default: false)" - type = bool - default = false -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/versions.tf deleted file mode 100644 index 67eb446cde..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/project/service-enablement/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:service-enablement/v1.57.0" - } - - required_version = ">= 0.14.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/README.md deleted file mode 100644 index 052e6aee23..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# Description - -This module creates a Bigquery Pub/Sub Subscription. - -Primarily used for FSI - MonteCarlo Tutorial: -**[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -## Example - -The following example creates a Bigquery subscription using a Bigquery table and -Pub/Sub topic. - -```yaml - - id: bq_subscription - source: community/modules/pubsub/bigquery-sub - use: [bq-table, pubsub_topic] -``` - -Also see usages in this -[example blueprint](../../../examples/fsi-montecarlo-on-batch.yaml). - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_project_iam_member.editor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | -| [google_project_iam_member.viewer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | -| [google_pubsub_subscription.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [dataset\_id](#input\_dataset\_id) | Name of the dataset that was created. Can be provided by the bigquery-table module | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [subscription\_id](#input\_subscription\_id) | The name of the pubsub subscription to be created | `string` | `null` | no | -| [table\_id](#input\_table\_id) | ID of created BQ table. Can be provided by the bigquery-table module | `string` | n/a | yes | -| [topic\_id](#input\_topic\_id) | The name of the pubsub topic to subscribe to. Can be provided by the pubsub/topic module | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [subscription\_id](#output\_subscription\_id) | Name of the subscription that was created. | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf deleted file mode 100644 index 8edbc6b24e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/main.tf +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "bigquery-sub", ghpc_role = "pubsub" }) -} - -locals { - subscription_id = var.subscription_id != null ? var.subscription_id : "${var.deployment_name}_subscription_${random_id.resource_name_suffix.hex}" -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} -data "google_project" "project" { - project_id = var.project_id -} - -resource "google_project_iam_member" "viewer" { - project = data.google_project.project.project_id - role = "roles/bigquery.metadataViewer" - member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" -} - -resource "google_project_iam_member" "editor" { - project = data.google_project.project.project_id - role = "roles/bigquery.dataEditor" - member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-pubsub.iam.gserviceaccount.com" -} - -resource "google_pubsub_subscription" "example" { - depends_on = [google_project_iam_member.editor, google_project_iam_member.viewer] - name = local.subscription_id - topic = var.topic_id - project = var.project_id - labels = local.labels - bigquery_config { - table = "${var.project_id}.${var.dataset_id}.${var.table_id}" - use_topic_schema = true - write_metadata = true - } - -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml deleted file mode 100644 index 9aedef48dc..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - pubsub.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf deleted file mode 100644 index fc81859503..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/outputs.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "subscription_id" { - description = "Name of the subscription that was created." - value = google_pubsub_subscription.example.name -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf deleted file mode 100644 index ee4dbbed8e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/variables.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "topic_id" { - description = "The name of the pubsub topic to subscribe to. Can be provided by the pubsub/topic module" - type = string -} - -variable "subscription_id" { - description = "The name of the pubsub subscription to be created" - type = string - default = null -} - -variable "dataset_id" { - description = "Name of the dataset that was created. Can be provided by the bigquery-table module" - type = string -} - -variable "table_id" { - description = "ID of created BQ table. Can be provided by the bigquery-table module" - type = string -} - -variable "labels" { - description = "Labels to add to the instances. Key-value pairs." - type = map(string) -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf deleted file mode 100644 index 54f9ccf8bb..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/bigquery-sub/versions.tf +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.57.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:bigquery-sub/v1.57.0" - } - required_version = ">= 1.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/README.md deleted file mode 100644 index 177f799dc6..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/README.md +++ /dev/null @@ -1,82 +0,0 @@ -## Description - -Creates a Pub/Sub topic - -Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**. - -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -### Example - -The following example creates a Pub/Sub topic. - -```yaml - - id: pubsub_topic - source: community/modules/pubsub/topic -``` - -Also see usages in this -[example blueprint](../../../examples/fsi-montecarlo-on-batch.yaml). - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_pubsub_schema.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_schema) | resource | -| [google_pubsub_topic.example](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_topic) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [schema\_id](#input\_schema\_id) | The name of the pubsub schema to be created | `string` | `null` | no | -| [schema\_json](#input\_schema\_json) | The JSON definition of the pubsub topic schema | `string` | `"{ \n \"name\" : \"Avro\", \n \"type\" : \"record\", \n \"fields\" : \n [\n {\"name\" : \"ticker\", \"type\" : \"string\"},\n {\"name\" : \"epoch_time\", \"type\" : \"int\"},\n {\"name\" : \"iteration\", \"type\" : \"int\"},\n {\"name\" : \"start_date\", \"type\" : \"string\"},\n {\"name\" : \"end_date\", \"type\" : \"string\"},\n {\n \"name\":\"simulation_results\",\n \"type\":{\n \"type\": \"array\", \n \"items\":{\n \"name\":\"Child\",\n \"type\":\"record\",\n \"fields\":[\n {\"name\":\"price\", \"type\":\"double\"}\n ]\n }\n }\n }\n ]\n }\n"` | no | -| [topic\_id](#input\_topic\_id) | The name of the pubsub topic to be created | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [topic\_id](#output\_topic\_id) | Name of the topic that was created. | -| [topic\_schema](#output\_topic\_schema) | Name of the topic schema that was created. | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/main.tf deleted file mode 100644 index 4ba68fb5d0..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/main.tf +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "topic", ghpc_role = "pubsub" }) -} - -locals { - topic_id = var.topic_id != null ? var.topic_id : "${var.deployment_name}_topic_${random_id.resource_name_suffix.hex}" - schema_id = var.schema_id != null ? var.schema_id : "${var.deployment_name}_schema_${random_id.resource_name_suffix.hex}" -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_pubsub_topic" "example" { - name = local.topic_id - depends_on = [google_pubsub_schema.example] - project = var.project_id - labels = local.labels - schema_settings { - schema = "projects/${var.project_id}/schemas/${local.schema_id}" - encoding = "BINARY" - } -} - -resource "google_pubsub_schema" "example" { - name = local.schema_id - project = var.project_id - type = "AVRO" - - definition = var.schema_json -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/metadata.yaml deleted file mode 100644 index 9aedef48dc..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - pubsub.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/outputs.tf deleted file mode 100644 index 3ea9d951b2..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/outputs.tf +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "topic_id" { - description = "Name of the topic that was created." - value = google_pubsub_topic.example.name -} - - -output "topic_schema" { - description = "Name of the topic schema that was created." - value = local.schema_id -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/variables.tf deleted file mode 100644 index dca575d21d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/pubsub/topic/variables.tf +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "topic_id" { - description = "The name of the pubsub topic to be created" - type = string - default = null -} - -variable "schema_id" { - description = "The name of the pubsub schema to be created" - type = string - default = null -} - -variable "schema_json" { - description = "The JSON definition of the pubsub topic schema" - type = string - default = < **Note**: This is an experimental module. This module has only been tested in -> limited capacity with the Cluster Toolkit. The module interface may have undergo -> breaking changes in the future. - -### Example - -The following example will create a single GPU accelerated remote desktop. - -```yaml - - id: remote-desktop - source: community/modules/remote-desktop/chrome-remote-desktop - use: [network1] - settings: - install_nvidia_driver: true -``` - -### Setting up the Remote Desktop - -1. Once the remote desktop has been deployed, navigate to https://remotedesktop.google.com/headless. -1. Click through `Begin`, `Next`, & `Authorize`. -1. Copy the code snippet for `Debian Linux`. -1. SSH into the remote desktop machine. It will be listed under - [VM Instances](https://console.cloud.google.com/compute/instances) in the - Google Cloud web console. -1. Run the copied command and follow instructions to set up a PIN. -1. You should now see your machine listed on the - [Chrome Remote Desktop page](https://remotedesktop.google.com/access) under `Remote devices`. -1. Click on your machine and enter PIN if prompted. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.12.31 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [client\_startup\_script](#module\_client\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | -| [instances](#module\_instances) | ../../../../modules/compute/vm-instance | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [add\_deployment\_name\_before\_prefix](#input\_add\_deployment\_name\_before\_prefix) | If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments.
See `name_prefix` for further details on resource naming behavior. | `bool` | `false` | no | -| [auto\_delete\_boot\_disk](#input\_auto\_delete\_boot\_disk) | Controls if boot disk should be auto-deleted when instance is deleted. | `bool` | `true` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image th
at supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-v
m-with-high-bandwidth-configuration) for more details. | `string` | `"not_enabled"` | no | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. Cloud resource names will include this value. | `string` | n/a | yes | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for instances. | `number` | `200` | no | -| [disk\_type](#input\_disk\_type) | Disk type for instances. | `string` | `"pd-balanced"` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true, instances will have public IPs on the internet. | `bool` | `true` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. Requires virtual workstation accelerator if Nvidia Grid Drivers are required | 
list(object({
    type  = string,
    count = number
  }))
| 
[
  {
    "count": 1,
    "type": "nvidia-tesla-t4-vws"
  }
]
| no | -| [install\_nvidia\_driver](#input\_install\_nvidia\_driver) | Installs the nvidia driver (true/false). For details, see https://cloud.google.com/compute/docs/gpus/install-drivers-gpu | `bool` | n/a | yes | -| [instance\_count](#input\_instance\_count) | Number of instances | `number` | `1` | no | -| [instance\_image](#input\_instance\_image) | Image used to build chrome remote desktop node. The default image is
name="debian-12-bookworm-v20250610" and project="debian-cloud".
NOTE: uses fixed version of image to avoid NVIDIA driver compatibility issues.

An alternative image is from name="ubuntu-2204-jammy-v20240126" and project="ubuntu-os-cloud".

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "name": "debian-12-bookworm-v20250610",
  "project": "debian-cloud"
}
| no | -| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation. Must be N1 family if GPU is used. | `string` | `"n1-standard-8"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | -| [name\_prefix](#input\_name\_prefix) | An optional name for all VM and disk resources.
If not supplied, `deployment_name` will be used.
When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set,
then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". | `string` | `null` | no | -| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces. The options match that of the terraform
network\_interface block of google\_compute\_instance. For descriptions of the
subfields or more information see the documentation:
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface
**\_NOTE:\_** If `network_interfaces` are set, `network_self_link` and
`subnetwork_self_link` will be ignored, even if they are provided through
the `use` field. `bandwidth_tier` and `enable_public_ips` also do not apply
to network interfaces defined in this variable.
Subfields:
network (string, required if subnetwork is not supplied)
subnetwork (string, required if network is not supplied)
subnetwork\_project (string, optional)
network\_ip (string, optional)
nic\_type (string, optional, choose from ["GVNIC", "VIRTIO\_NET", "RDMA", "IRDMA", "MRDMA"])
stack\_type (string, optional, choose from ["IPV4\_ONLY", "IPV4\_IPV6"])
queue\_count (number, optional)
access\_config (object, optional)
ipv6\_access\_config (object, optional)
alias\_ip\_range (list(object), optional) | 
list(object({
    network            = string,
    subnetwork         = string,
    subnetwork_project = string,
    network_ip         = string,
    nic_type           = string,
    stack_type         = string,
    queue_count        = number,
    access_config = list(object({
      nat_ip                 = string,
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    ipv6_access_config = list(object({
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    alias_ip_range = list(object({
      ip_cidr_range         = string,
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the VM. | `string` | `"default"` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE` | `string` | `"TERMINATE"` | no | -| [project\_id](#input\_project\_id) | Project in which Google Cloud resources will be created | `string` | n/a | yes | -| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | -| [service\_account](#input\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account. | 
object({
    email  = string,
    scopes = set(string)
  })
| 
{
  "email": null,
  "scopes": [
    "https://www.googleapis.com/auth/cloud-platform"
  ]
}
| no | -| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | -| [startup\_script](#input\_startup\_script) | Startup script used on the instance | `string` | `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the VM. | `string` | `null` | no | -| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | -| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core | `number` | `2` | no | -| [zone](#input\_zone) | Default zone for creating resources | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [instance\_name](#output\_instance\_name) | Name of the first instance created, if any. | -| [startup\_script](#output\_startup\_script) | script to load and run all runners, as a string value. | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf deleted file mode 100644 index a5cf7c5d37..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/main.tf +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "chrome-remote-desktop", ghpc_role = "remote-desktop" }) -} - -locals { - - user_startup_script_runners = var.startup_script == null ? [] : [ - { - type = "shell" - content = var.startup_script - destination = "user_startup_script.sh" - } - ] - - configure_nvidia_driver_runners = var.install_nvidia_driver == false ? [] : [ - { - type = "ansible-local" - content = file("${path.module}/scripts/configure-grid-drivers.yml") - destination = "/usr/local/ghpc/configure-grid-drivers.yml" - } - ] - - configure_chrome_remote_desktop_runners = [ - { - type = "ansible-local" - content = file("${path.module}/scripts/configure-chrome-desktop.yml") - destination = "/usr/local/ghpc/configure-chrome-desktop.yml" - } - ] - - disable_sleep = [ - { - type = "ansible-local" - content = file("${path.module}/scripts/disable-sleep.yml") - destination = "/usr/local/ghpc/disable-sleep.yml" - } - ] -} - -module "client_startup_script" { - source = "../../../../modules/scripts/startup-script" - - deployment_name = var.deployment_name - project_id = var.project_id - region = var.region - labels = local.labels - - runners = flatten([ - local.user_startup_script_runners, - local.configure_nvidia_driver_runners, - local.configure_chrome_remote_desktop_runners, - local.disable_sleep - ]) -} - -module "instances" { - source = "../../../../modules/compute/vm-instance" - - instance_count = var.instance_count - name_prefix = var.name_prefix - add_deployment_name_before_prefix = var.add_deployment_name_before_prefix - provisioning_model = var.spot ? "SPOT" : null - - deployment_name = var.deployment_name - project_id = var.project_id - region = var.region - zone = var.zone - labels = local.labels - - machine_type = var.machine_type - service_account_email = var.service_account.email - metadata = var.metadata - startup_script = module.client_startup_script.startup_script - enable_oslogin = var.enable_oslogin - - instance_image = var.instance_image - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - auto_delete_boot_disk = var.auto_delete_boot_disk - - disable_public_ips = !var.enable_public_ips - network_self_link = var.network_self_link - subnetwork_self_link = var.subnetwork_self_link - network_interfaces = var.network_interfaces - bandwidth_tier = var.bandwidth_tier - tags = var.tags - - threads_per_core = var.threads_per_core - guest_accelerator = var.guest_accelerator - on_host_maintenance = var.on_host_maintenance - - network_storage = var.network_storage - -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf deleted file mode 100644 index bcf8ece52d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "script to load and run all runners, as a string value." - value = module.client_startup_script.startup_script -} - -output "instance_name" { - description = "Name of the first instance created, if any." - value = var.instance_count > 0 ? module.instances.name[0] : null -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml deleted file mode 100644 index 391aa86433..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Ensure Desktop OS and Chrome Remote Desktop is installed - hosts: localhost - become: true - module_defaults: - ansible.builtin.apt: - update_cache: true - cache_valid_time: 3600 - tasks: - - name: Install desktop packages - ansible.builtin.apt: - name: - - xfce4 - - xfce4-goodies - state: present - register: apt_result - retries: 10 - delay: 30 - until: apt_result is success - - - name: Download and configure CRD - ansible.builtin.get_url: - url: https://dl.google.com/linux/direct/chrome-remote-desktop_current_amd64.deb - dest: /tmp/chrome-remote-desktop_current_amd64.deb - mode: "0755" - timeout: 30 - - - name: Install CRD - ansible.builtin.apt: - deb: /tmp/chrome-remote-desktop_current_amd64.deb - environment: - DEBIAN_FRONTEND: noninteractive - register: apt_result - retries: 10 - delay: 30 - until: apt_result is success - - - name: Configure CRD to use Xfce by default - ansible.builtin.copy: - dest: /etc/chrome-remote-desktop-session - content: "exec /etc/X11/Xsession /usr/bin/xfce4-session" - mode: 0644 - - - name: Start Chrome remote desktop - ansible.builtin.command: /etc/init.d/chrome-remote-desktop start - register: result - changed_when: result.rc == 0 diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml deleted file mode 100644 index daae08176d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Ensure nvidia grid drivers and other binaries are installed - hosts: localhost - become: true - vars: - dist_settings: - bullseye: - packages: - - build-essential - - gdebi-core - - mesa-utils - - gdm3 - - linux-headers-{{ ansible_kernel }} - grid_fn: NVIDIA-Linux-x86_64-510.85.02-grid.run - grid_ver: vGPU14.2 - bookworm: - packages: - - build-essential - - gdebi-core - - mesa-utils - - gdm3 - - linux-headers-{{ ansible_kernel }} - grid_fn: NVIDIA-Linux-x86_64-550.54.15-grid.run - grid_ver: vGPU17.1 - jammy: - packages: - - build-essential - - gdebi-core - - mesa-utils - - gdm3 - - gcc-12 # must match compiler used to build kernel on latest Ubuntu 22 - - pkg-config # observed to be necessary for GRID driver installation on latest Ubuntu 22 - - libglvnd-dev # observed to be necessary for GRID driver installation on latest Ubuntu 22 - - linux-headers-{{ ansible_kernel }} - grid_fn: NVIDIA-Linux-x86_64-525.125.06-grid.run - grid_ver: vGPU15.3 - tasks: - - name: Fail if using wrong OS - ansible.builtin.assert: - that: - - ansible_os_family in ["Debian", "Ubuntu"] - - ansible_distribution_release in dist_settings.keys() | list - fail_msg: "ansible_os_family: {{ ansible_os_family }} or ansible_distribution_release: {{ansible_distribution_release}} was not acceptable." - - - name: Check if GRID driver installed - ansible.builtin.command: which nvidia-smi - register: nvidiasmi_result - ignore_errors: true - changed_when: false - - - name: Install binaries for GRID drivers - ansible.builtin.apt: - name: '{{ dist_settings[ansible_distribution_release]["packages"] }}' - state: present - update_cache: true - register: apt_result - retries: 6 - delay: 10 - until: apt_result is success - - - name: Install GRID driver if not existing - when: nvidiasmi_result is failed - block: - - name: Download GPU driver - ansible.builtin.get_url: - url: https://storage.googleapis.com/nvidia-drivers-us-public/GRID/{{ dist_settings[ansible_distribution_release]["grid_ver"] }}/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} - dest: /tmp/ - mode: "0755" - timeout: 30 - - - name: Stop gdm service - ansible.builtin.systemd: - name: gdm - state: stopped - - - name: Install GPU driver - ansible.builtin.shell: | - #jinja2: trim_blocks: "True" - {% if ansible_distribution_release == "jammy" %} - CC=gcc-12 /tmp/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} --silent - {% else %} - /tmp/{{ dist_settings[ansible_distribution_release]["grid_fn"] }} --silent - {% endif %} - register: result - changed_when: result.rc == 0 - - - name: Download VirtualGL driver - ansible.builtin.get_url: - url: https://sourceforge.net/projects/virtualgl/files/3.0.2/virtualgl_3.0.2_amd64.deb/download - dest: /tmp/virtualgl_3.0.2_amd64.deb - mode: "0755" - timeout: 30 - - - name: Install VirtualGL - ansible.builtin.command: gdebi /tmp/virtualgl_3.0.2_amd64.deb --non-interactive - register: result - changed_when: result.rc == 0 - - - name: Fix headless Nvidia issue - block: - - name: Lookup gpu info - ansible.builtin.command: nvidia-xconfig --query-gpu-info - register: gpu_info - failed_when: gpu_info.rc != 0 - changed_when: false - - - name: Extract PCI ID - ansible.builtin.shell: | - set -o pipefail - echo "{{ gpu_info.stdout }}" | grep "PCI BusID " | head -n 1 | cut -d':' -f2-99 | xargs - args: - executable: /bin/bash - register: pci_id - changed_when: false - - - name: Configure nvidia-xconfig - ansible.builtin.command: nvidia-xconfig -a --allow-empty-initial-configuration --enable-all-gpus --virtual=1920x1200 --busid={{ pci_id.stdout }} - register: result - changed_when: result.rc == 0 - - - name: Set HardDPMS to false - ansible.builtin.replace: - path: /etc/X11/xorg.conf - regexp: "Section \"Device\"" - replace: "Section \"Device\"\n Option \"HardDPMS\" \"false\"" - - - name: Configure VirtualGL for X - ansible.builtin.command: vglserver_config +glx +s +f -t - register: result - changed_when: result.rc == 0 - - - name: Configure gdm for X - block: - - name: Configure default display manager - ansible.builtin.copy: - dest: /etc/X11/default-display-manager - content: "/usr/sbin/gdm3" - mode: 0644 - - - name: Switch boot target to gui - ansible.builtin.command: systemctl set-default graphical.target - register: result - changed_when: result.rc == 0 - - - name: Start gdm service - ansible.builtin.systemd: - name: gdm - daemon_reload: true - state: started diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml deleted file mode 100644 index 6767b05fb2..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/scripts/disable-sleep.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Mask sleep, suspend, hibernate, and hybrid-sleep targets - hosts: localhost - become: true - tasks: - - - name: Mask sleep target - ansible.builtin.systemd: - name: sleep.target - masked: true - - - name: Mask suspend target - ansible.builtin.systemd: - name: suspend.target - masked: true - - - name: Mask hibernate target - ansible.builtin.systemd: - name: hibernate.target - masked: true - - - name: Mask hybrid-sleep target - ansible.builtin.systemd: - name: hybrid-sleep.target - masked: true diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf deleted file mode 100644 index ac4c3b1869..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/remote-desktop/chrome-remote-desktop/variables.tf +++ /dev/null @@ -1,277 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which Google Cloud resources will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. Cloud resource names will include this value." - type = string - #default = "chrome-remote-desktop" -} - -variable "region" { - description = "Default region for creating resources" - type = string -} - -variable "zone" { - description = "Default zone for creating resources" - type = string -} - -variable "instance_count" { - description = "Number of instances" - type = number - default = 1 -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured." - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "instance_image" { - description = <<-EOD - Image used to build chrome remote desktop node. The default image is - name="debian-12-bookworm-v20250610" and project="debian-cloud". - NOTE: uses fixed version of image to avoid NVIDIA driver compatibility issues. - - An alternative image is from name="ubuntu-2204-jammy-v20240126" and project="ubuntu-os-cloud". - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "debian-cloud" - name = "debian-12-bookworm-v20250610" - } -} - -variable "disk_size_gb" { - description = "Size of disk for instances." - type = number - default = 200 -} - -variable "disk_type" { - description = "Disk type for instances." - type = string - default = "pd-balanced" -} - -variable "auto_delete_boot_disk" { - description = "Controls if boot disk should be auto-deleted when instance is deleted." - type = bool - default = true -} - -variable "name_prefix" { - description = <<-EOT - An optional name for all VM and disk resources. - If not supplied, `deployment_name` will be used. - When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set, - then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". - EOT - type = string - default = null -} - -variable "add_deployment_name_before_prefix" { - description = <<-EOT - If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments. - See `name_prefix` for further details on resource naming behavior. - EOT - type = bool - default = false -} - -variable "enable_public_ips" { - description = "If set to true, instances will have public IPs on the internet." - type = bool - default = true -} - -variable "machine_type" { - description = "Machine type to use for the instance creation. Must be N1 family if GPU is used." - type = string - default = "n1-standard-8" -} - -variable "labels" { - description = "Labels to add to the instances. Key-value pairs." - type = map(string) - default = {} -} - -variable "service_account" { - description = "Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account." - type = object({ - email = string, - scopes = set(string) - }) - default = { - email = null - scopes = [ - "https://www.googleapis.com/auth/cloud-platform", - ] - } -} - -variable "network_self_link" { - description = "The self link of the network to attach the VM." - type = string - default = "default" -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork to attach the VM." - type = string - default = null -} - -variable "network_interfaces" { - description = <<-EOT - A list of network interfaces. The options match that of the terraform - network_interface block of google_compute_instance. For descriptions of the - subfields or more information see the documentation: - https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface - **_NOTE:_** If `network_interfaces` are set, `network_self_link` and - `subnetwork_self_link` will be ignored, even if they are provided through - the `use` field. `bandwidth_tier` and `enable_public_ips` also do not apply - to network interfaces defined in this variable. - Subfields: - network (string, required if subnetwork is not supplied) - subnetwork (string, required if network is not supplied) - subnetwork_project (string, optional) - network_ip (string, optional) - nic_type (string, optional, choose from ["GVNIC", "VIRTIO_NET", "RDMA", "IRDMA", "MRDMA"]) - stack_type (string, optional, choose from ["IPV4_ONLY", "IPV4_IPV6"]) - queue_count (number, optional) - access_config (object, optional) - ipv6_access_config (object, optional) - alias_ip_range (list(object), optional) - EOT - type = list(object({ - network = string, - subnetwork = string, - subnetwork_project = string, - network_ip = string, - nic_type = string, - stack_type = string, - queue_count = number, - access_config = list(object({ - nat_ip = string, - public_ptr_domain_name = string, - network_tier = string - })), - ipv6_access_config = list(object({ - public_ptr_domain_name = string, - network_tier = string - })), - alias_ip_range = list(object({ - ip_cidr_range = string, - subnetwork_range_name = string - })) - })) - default = [] -} - -variable "metadata" { - description = "Metadata, provided as a map" - type = map(string) - default = {} -} - -variable "startup_script" { - description = "Startup script used on the instance" - type = string - default = null -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance. Requires virtual workstation accelerator if Nvidia Grid Drivers are required" - type = list(object({ - type = string, - count = number - })) - default = [{ - type = "nvidia-tesla-t4-vws" - count = 1 - }] -} - -variable "threads_per_core" { - description = "Sets the number of threads per physical core" - type = number - default = 2 -} - -variable "on_host_maintenance" { - description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE`" - type = string - default = "TERMINATE" -} - -variable "bandwidth_tier" { - description = <> --all-instances --region <> \ - --project <> --minimal-action replace -``` - -This mode can be switched to proactive (automatic) replacement by setting -[var.update_policy](#input_update_policy) to "PROACTIVE". In this case we -recommend the use of Filestore to store the job queue state ("spool") and -setting [var.spool_parent_dir][#input_spool_parent_dir] to its mount point: - -```yaml - - id: spoolfs - source: modules/file-system/filestore - use: - - network1 - settings: - filestore_tier: ENTERPRISE - local_mount: /shared - -... - - - id: htcondor_access - source: community/modules/scheduler/htcondor-access-point - use: - - network1 - - spoolfs - - htcondor_secrets - - htcondor_setup - - htcondor_cm - - htcondor_execute_point_group - settings: - spool_parent_dir: /shared -``` - -[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.1 | -| [google](#requirement\_google) | >= 3.83 | -| [null](#requirement\_null) | >= 3.0 | -| [random](#requirement\_random) | ~> 3.6 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [null](#provider\_null) | >= 3.0 | -| [random](#provider\_random) | ~> 3.6 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [access\_point\_instance\_template](#module\_access\_point\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | -| [htcondor\_ap](#module\_htcondor\_ap) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_compute_address.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | -| [google_compute_disk.spool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [google_compute_region_disk.spool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_region_disk) | resource | -| [google_storage_bucket_object.ap_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [null_resource.ap_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [random_shuffle.zones](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/shuffle) | resource | -| [google_compute_image.htcondor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | -| [google_compute_instance.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance) | data source | -| [google_compute_region_instance_group.ap](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_group) | data source | -| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_point\_runner](#input\_access\_point\_runner) | A list of Toolkit runners for configuring an HTCondor access point | `list(map(string))` | `[]` | no | -| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | Service account for access point (e-mail format) | `string` | n/a | yes | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [autoscaler\_runner](#input\_autoscaler\_runner) | A list of Toolkit runners for configuring autoscaling daemons | `list(map(string))` | `[]` | no | -| [central\_manager\_ips](#input\_central\_manager\_ips) | List of IP addresses of HTCondor Central Managers | `list(string)` | n/a | yes | -| [default\_mig\_id](#input\_default\_mig\_id) | Default MIG ID for HTCondor jobs; if unset, jobs must specify MIG id | `string` | `""` | no | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `32` | no | -| [disk\_type](#input\_disk\_type) | Boot disk size in GB | `string` | `"pd-balanced"` | no | -| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape acoss zones for instance group managing high availability of access point | `string` | `"ANY_SINGLE_ZONE"` | no | -| [enable\_high\_availability](#input\_enable\_high\_availability) | Provision HTCondor access point in high availability mode | `bool` | `false` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [enable\_public\_ips](#input\_enable\_public\_ips) | Enable Public IPs on the access points | `bool` | `false` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | -| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | -| [instance\_image](#input\_instance\_image) | Custom VM image with HTCondor and Toolkit support installed."

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | n/a | yes | -| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | -| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor central managers | `string` | `"n2-standard-4"` | no | -| [metadata](#input\_metadata) | Metadata to add to HTCondor central managers | `map(string)` | `{}` | no | -| [mig\_id](#input\_mig\_id) | List of Managed Instance Group IDs containing execute points in this pool (supplied by htcondor-execute-point module) | `list(string)` | `[]` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network in which the HTCondor central manager will be created. | `string` | `null` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | -| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [spool\_disk\_size\_gb](#input\_spool\_disk\_size\_gb) | Boot disk size in GB | `number` | `32` | no | -| [spool\_disk\_type](#input\_spool\_disk\_type) | Boot disk size in GB | `string` | `"pd-ssd"` | no | -| [spool\_parent\_dir](#input\_spool\_parent\_dir) | HTCondor access point configuration SPOOL will be set to subdirectory named "spool" | `string` | `"/var/lib/condor"` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which the HTCondor central manager will be created. | `string` | `null` | no | -| [update\_policy](#input\_update\_policy) | Replacement policy for Access Point Managed Instance Group ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle) | `string` | `"OPPORTUNISTIC"` | no | -| [zones](#input\_zones) | Zone(s) in which access point may be created. If not supplied, defaults to 2 randomly-selected zones in var.region. | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [access\_point\_ips](#output\_access\_point\_ips) | IP addresses of the access points provisioned by this module | -| [access\_point\_name](#output\_access\_point\_name) | Name of the access point provisioned by this module | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml deleted file mode 100644 index 6a2f50c831..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/files/htcondor_configure.yml +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Configure HTCondor Access Point - hosts: localhost - become: true - vars: - spool_dir: /var/lib/condor/spool - condor_config_root: /etc/condor - ghpc_config_file: 50-ghpc-managed - htcondor_spool_disk_device: /dev/disk/by-id/google-htcondor-spool-disk - tasks: - - name: Ensure necessary variables are set - ansible.builtin.assert: - that: - - htcondor_role is defined - - config_object is defined - - name: Remove default HTCondor configuration - ansible.builtin.file: - path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" - state: absent - notify: - - Reload HTCondor - - name: Create Toolkit configuration file - register: config_update - changed_when: config_update.rc == 137 - failed_when: config_update.rc != 0 and config_update.rc != 137 - ansible.builtin.shell: | - set -e -o pipefail - REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) - - CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" - if [ -f "${CONFIG_FILE}" ]; then - LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") - else - LOCAL_HASH="INVALID-HASH" - fi - - if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then - gcloud storage cp {{ config_object }} "${CONFIG_FILE}" - chmod 0644 "${CONFIG_FILE}" - exit 137 - fi - args: - executable: /bin/bash - notify: - - Reload HTCondor - - name: Configure HTCondor SchedD - when: htcondor_role == 'get_htcondor_submit' - block: - - name: Format spool disk - community.general.filesystem: - fstype: ext4 - state: present - dev: "{{ htcondor_spool_disk_device }}" - # RUN TUNE2FS - - name: Mount spool (creates mount point) - ansible.posix.mount: - path: "{{ spool_dir }}" - src: "{{ htcondor_spool_disk_device }}" - fstype: ext4 - opts: defaults - state: mounted - - name: Ensure spool free space - ansible.builtin.command: tune2fs -r 0 {{ htcondor_spool_disk_device }} - - name: Setup spool directory - ansible.builtin.file: - path: "{{ spool_dir }}" - state: directory - owner: condor - group: condor - mode: 0755 - recurse: true - - name: Create SystemD override directory for HTCondor - ansible.builtin.file: - path: /etc/systemd/system/condor.service.d - state: directory - owner: root - group: root - mode: 0755 - - name: Ensure HTCondor starts after shared filesystem is mounted - ansible.builtin.copy: - dest: /etc/systemd/system/condor.service.d/mount-spool.conf - mode: 0644 - content: | - [Unit] - RequiresMountsFor={{ spool_dir }} - notify: - - Reload SystemD - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true - - name: Reload HTCondor - ansible.builtin.service: - name: condor - state: reloaded - post_tasks: - - name: Start HTCondor - ansible.builtin.service: - name: condor - state: started - enabled: true - - name: Inform users - changed_when: false - ansible.builtin.shell: | - set -e -o pipefail - wall "******* HTCondor configuration complete; startup-script may still be executing ********" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf deleted file mode 100644 index fdbcf5c32f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/main.tf +++ /dev/null @@ -1,338 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "htcondor-access-point", ghpc_role = "scheduler" }) -} - -locals { - network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } - oslogin_api_values = { - "DISABLE" = "FALSE" - "ENABLE" = "TRUE" - } - enable_oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - metadata = merge( - local.network_storage_metadata, - local.enable_oslogin_metadata, - local.disable_automatic_updates_metadata, - var.metadata - ) - - host_count = 1 - name_prefix = "${var.deployment_name}-ap" - - example_runner = { - type = "data" - destination = "/var/tmp/helloworld.sub" - content = <<-EOT - universe = vanilla - executable = /bin/sleep - arguments = 1000 - output = out.$(ClusterId).$(ProcId) - error = err.$(ClusterId).$(ProcId) - log = log.$(ClusterId).$(ProcId) - request_cpus = 1 - request_memory = 100MB - queue - EOT - } - - native_fstype = [] - startup_script_network_storage = [ - for ns in var.network_storage : - ns if !contains(local.native_fstype, ns.fs_type) - ] - storage_client_install_runners = [ - for ns in local.startup_script_network_storage : - ns.client_install_runner if ns.client_install_runner != null - ] - mount_runners = [ - for ns in local.startup_script_network_storage : - ns.mount_runner if ns.mount_runner != null - ] - - all_runners = concat( - local.storage_client_install_runners, - local.mount_runners, - var.access_point_runner, - [local.schedd_runner], - var.autoscaler_runner, - [local.example_runner] - ) - - ap_config = templatefile("${path.module}/templates/condor_config.tftpl", { - htcondor_role = "get_htcondor_submit", - central_manager_ips = var.central_manager_ips - spool_dir = "${var.spool_parent_dir}/spool", - mig_ids = var.mig_id, - default_mig_id = var.default_mig_id - }) - - ap_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.ap_config.output_name}" - schedd_runner = { - type = "ansible-local" - content = file("${path.module}/files/htcondor_configure.yml") - destination = "htcondor_configure.yml" - args = join(" ", [ - "-e htcondor_role=get_htcondor_submit", - "-e config_object=${local.ap_object}", - "-e spool_dir=${var.spool_parent_dir}/spool", - "-e htcondor_spool_disk_device=/dev/disk/by-id/google-${local.spool_disk_device_name}", - ]) - } - - access_point_ips = google_compute_address.ap.address - access_point_name = data.google_compute_instance.ap.name - - spool_disk_resource_name = "${var.deployment_name}-spool-disk" - spool_disk_device_name = "htcondor-spool-disk" - spool_disk_source = try(google_compute_disk.spool[0].name, google_compute_region_disk.spool[0].self_link) - - zones = coalescelist(var.zones, random_shuffle.zones.result) - - vm_family = split("-", var.machine_type)[0] - regional_pd_families = ["e2", "n1", "n2", "n2d"] -} - -data "google_compute_image" "htcondor" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = var.instance_image.project - - lifecycle { - postcondition { - condition = self.disk_size_gb <= var.disk_size_gb - error_message = "var.disk_size_gb must be set to at least the size of the image (${self.disk_size_gb})" - } - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} - -data "google_compute_zones" "available" { - project = var.project_id - region = var.region - - lifecycle { - postcondition { - condition = alltrue([ - for z in var.zones : contains(self.names, z) - ]) - error_message = "Each entry in var.zones must be a zone in var.region: ${var.region}" - } - } -} - -resource "random_shuffle" "zones" { - input = data.google_compute_zones.available.names - result_count = var.enable_high_availability ? 2 : 1 -} - -data "google_compute_region_instance_group" "ap" { - self_link = module.htcondor_ap.self_link - lifecycle { - postcondition { - condition = length(self.instances) == local.host_count - error_message = "There should be ${local.host_count} access points found" - } - } -} - -data "google_compute_instance" "ap" { - self_link = data.google_compute_region_instance_group.ap.instances[0].instance -} - -resource "null_resource" "ap_config" { - triggers = { - config = local.ap_config - } -} - -resource "google_storage_bucket_object" "ap_config" { - name = "${local.name_prefix}-config-${substr(md5(null_resource.ap_config.id), 0, 4)}" - content = local.ap_config - bucket = var.htcondor_bucket_name - - lifecycle { - precondition { - condition = var.default_mig_id == "" || contains(var.mig_id, var.default_mig_id) - error_message = "If set, var.default_mig_id must be an element in var.mig_id" - } - - # by construction, this precondition only fails when the user has set - # var.zones to a non-empty list of length not equal to 2 - precondition { - condition = !var.enable_high_availability || length(local.zones) == 2 - error_message = "When using HTCondor access point high availability, var.zones must be of length 2." - } - } -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - project_id = var.project_id - region = var.region - labels = local.labels - deployment_name = var.deployment_name - - runners = local.all_runners -} - -resource "google_compute_region_disk" "spool" { - count = var.enable_high_availability ? 1 : 0 - name = local.spool_disk_resource_name - labels = local.labels - type = var.spool_disk_type - region = var.region - size = var.spool_disk_size_gb - - replica_zones = local.zones - - lifecycle { - precondition { - condition = var.spool_disk_size_gb >= 200 - error_message = "When using HTCondor access point high availability, var.spool_disk_size_gb must be set to 200 or greater." - } - - precondition { - condition = contains(local.regional_pd_families, local.vm_family) - error_message = "When using HTCondor access point high availability, var.machine_type must be one of ${jsonencode(local.regional_pd_families)}." - } - } -} - -resource "google_compute_disk" "spool" { - count = var.enable_high_availability ? 0 : 1 - name = local.spool_disk_resource_name - labels = local.labels - type = var.spool_disk_type - zone = local.zones[0] - size = var.spool_disk_size_gb -} - -resource "google_compute_address" "ap" { - project = var.project_id - name = local.name_prefix - region = var.region - subnetwork = var.subnetwork_self_link - address_type = "INTERNAL" - purpose = "GCE_ENDPOINT" -} - -module "access_point_instance_template" { - source = "terraform-google-modules/vm/google//modules/instance_template" - version = "~> 12.1" - - name_prefix = local.name_prefix - project_id = var.project_id - network = var.network_self_link - subnetwork = var.subnetwork_self_link - service_account = { - email = var.access_point_service_account_email - scopes = var.service_account_scopes - } - labels = local.labels - - machine_type = var.machine_type - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - preemptible = false - startup_script = module.startup_script.startup_script - metadata = local.metadata - source_image = data.google_compute_image.htcondor.self_link - - # secure boot - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config - - network_ip = google_compute_address.ap.id - - # spool disk - additional_disks = [ - { - source = local.spool_disk_source - device_name = local.spool_disk_device_name - } - ] -} - -module "htcondor_ap" { - source = "terraform-google-modules/vm/google//modules/mig" - version = "~> 12.1" - - project_id = var.project_id - region = var.region - distribution_policy_target_shape = var.distribution_policy_target_shape - distribution_policy_zones = local.zones - target_size = local.host_count - hostname = local.name_prefix - instance_template = module.access_point_instance_template.self_link - - health_check_name = "health-${local.name_prefix}" - health_check = { - type = "tcp" - initial_delay_sec = 600 - check_interval_sec = 20 - healthy_threshold = 2 - timeout_sec = 8 - unhealthy_threshold = 3 - response = "" - proxy_header = "NONE" - port = 9618 - request = "" - request_path = "" - host = "" - enable_logging = true - } - - update_policy = [{ - instance_redistribution_type = "NONE" - replacement_method = "RECREATE" # preserves hostnames (necessary for PROACTIVE replacement) - max_surge_fixed = 0 # must be 0 to preserve hostnames - max_unavailable_fixed = length(local.zones) - max_surge_percent = null - max_unavailable_percent = null - min_ready_sec = 300 - minimal_action = "REPLACE" - type = var.update_policy - }] - - stateful_disks = [{ - device_name = local.spool_disk_device_name - delete_rule = "ON_PERMANENT_INSTANCE_DELETION" - }] - stateful_ips = var.enable_public_ips ? [{ - interface_name = "nic0" - delete_rule = "ON_PERMANENT_INSTANCE_DELETION" - is_external = true - }] : [] - - # the timeouts below are default for resource - wait_for_instances = true - mig_timeouts = { - create = "15m" - delete = "15m" - update = "15m" - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml deleted file mode 100644 index 3a78f9a46b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - storage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf deleted file mode 100644 index f7424c6d5d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "access_point_ips" { - description = "IP addresses of the access points provisioned by this module" - value = local.access_point_ips -} - -output "access_point_name" { - description = "Name of the access point provisioned by this module" - value = local.access_point_name -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl deleted file mode 100644 index 214fbc726f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/templates/condor_config.tftpl +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# this file is managed by the Cluster Toolkit; do not edit it manually -# override settings with a higher priority (last lexically) named file -# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration - -use role:${htcondor_role} -CONDOR_HOST = ${join(",", central_manager_ips)} - -SPOOL = ${spool_dir} -SCHEDD_INTERVAL = 30 -TRUST_UID_DOMAIN = True -SUBMIT_ATTRS = RunAsOwner -RunAsOwner = True - -# When a job matches to a machine, add machine attributes to the job for -# condor_history (e.g. VM Instance ID) -use feature:JobsHaveInstanceIDs -SYSTEM_JOB_MACHINE_ATTRS = $(SYSTEM_JOB_MACHINE_ATTRS) \ - CloudVMType CloudZone CloudInterruptible -SYSTEM_JOB_MACHINE_ATTRS_HISTORY_LENGTH = 10 - -# Add Cloud attributes to SchedD ClassAd -use feature:ScheddCronOneShot(cloud, $(LIBEXEC)/common-cloud-attributes-google.py) -SCHEDD_CRON_cloud_PREFIX = Cloud - -# aid the user by automatically using RequireSpot in their Requirements, unless -# the user has explicitly used CloudInterruptible -JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) SPOT -JOB_TRANSFORM_SPOT @=end - REQUIREMENTS ! isUndefined(RequireSpot) && ! unresolved(Requirements, "^CloudInterruptible$") - SET Requirements ($(MY.Requirements)) && (CloudInterruptible is My.RequireSpot) -@end - -# help the user by enforcing that RequireSpot is undefined or a boolean -SUBMIT_REQUIREMENT_NAMES = $(SUBMIT_REQUIREMENT_NAMES) SPOT -SUBMIT_REQUIREMENT_SPOT = isUndefined(RequireSpot) || isBoolean(RequireSpot) -SUBMIT_REQUIREMENT_SPOT_REASON = "If +RequireSpot is defined, it must be either True or False" - -%{ if length(mig_ids) > 0 ~} -MIG_IDS = "${join(" ", mig_ids)}" -MIG_ID_LIST = split($(MIG_IDS)) -%{ if default_mig_id != "" ~} -JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) ID_DEFAULT -JOB_TRANSFORM_ID_DEFAULT @=end - DEFAULT RequireId "${default_mig_id}" -@end -%{ endif ~} -SUBMIT_REQUIREMENT_NAMES = $(SUBMIT_REQUIREMENT_NAMES) MIGID -SUBMIT_REQUIREMENT_MIGID = !isUndefined(RequireId) && member(RequireId, $(MIG_ID_LIST)) -SUBMIT_REQUIREMENT_MIGID_REASON = strcat("Jobs must set +RequireId to one of following values surrounded by quotation marks:\n", $(MIG_IDS)) - -JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) MIGID -JOB_TRANSFORM_MIGID @=end - REQUIREMENTS ! isUndefined(RequireId) && ! unresolved(Requirements, "^CloudCreatedBy$") - SET Requirements ($(MY.Requirements)) && regexp(strcat("/", My.RequireId, "$"), CloudCreatedBy) -@end -%{ endif ~} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf deleted file mode 100644 index f54a88ac2e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/variables.tf +++ /dev/null @@ -1,266 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which HTCondor pool will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "labels" { - description = "Labels to add to resources. List key, value pairs." - type = map(string) -} - -variable "region" { - description = "Default region for creating resources" - type = string -} - -variable "zones" { - description = "Zone(s) in which access point may be created. If not supplied, defaults to 2 randomly-selected zones in var.region." - type = list(string) - default = [] - nullable = false - - validation { - condition = length(var.zones) <= 2 - error_message = "Set var.zones to the empty list or up to 2 zones in var.region" - } -} - -variable "distribution_policy_target_shape" { - description = "Target shape acoss zones for instance group managing high availability of access point" - type = string - default = "ANY_SINGLE_ZONE" -} - -variable "network_self_link" { - description = "The self link of the network in which the HTCondor central manager will be created." - type = string - default = null -} - -variable "access_point_service_account_email" { - description = "Service account for access point (e-mail format)" - type = string -} - -variable "service_account_scopes" { - description = "Scopes by which to limit service account attached to central manager." - type = set(string) - default = [ - "https://www.googleapis.com/auth/cloud-platform", - ] -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured" - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "disk_size_gb" { - description = "Boot disk size in GB" - type = number - default = 32 - nullable = false -} - -variable "disk_type" { - description = "Boot disk size in GB" - type = string - default = "pd-balanced" - nullable = false -} - -variable "spool_disk_size_gb" { - description = "Boot disk size in GB" - type = number - default = 32 - nullable = false -} - -variable "spool_disk_type" { - description = "Boot disk size in GB" - type = string - default = "pd-ssd" - nullable = false -} - -variable "metadata" { - description = "Metadata to add to HTCondor central managers" - type = map(string) - default = {} -} - -variable "enable_oslogin" { - description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." - type = string - default = "ENABLE" - nullable = false - validation { - condition = contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) - error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." - } -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork in which the HTCondor central manager will be created." - type = string - default = null -} - -variable "enable_high_availability" { - description = "Provision HTCondor access point in high availability mode" - type = bool - default = false -} - -variable "instance_image" { - description = <<-EOD - Custom VM image with HTCondor and Toolkit support installed." - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "machine_type" { - description = "Machine type to use for HTCondor central managers" - type = string - default = "n2-standard-4" -} - -variable "access_point_runner" { - description = "A list of Toolkit runners for configuring an HTCondor access point" - type = list(map(string)) - default = [] -} - -variable "autoscaler_runner" { - description = "A list of Toolkit runners for configuring autoscaling daemons" - type = list(map(string)) - default = [] -} - -variable "spool_parent_dir" { - description = "HTCondor access point configuration SPOOL will be set to subdirectory named \"spool\"" - type = string - default = "/var/lib/condor" -} - -variable "central_manager_ips" { - description = "List of IP addresses of HTCondor Central Managers" - type = list(string) -} - -variable "htcondor_bucket_name" { - description = "Name of HTCondor configuration bucket" - type = string -} - -variable "enable_public_ips" { - description = "Enable Public IPs on the access points" - type = bool - default = false -} - -variable "mig_id" { - description = "List of Managed Instance Group IDs containing execute points in this pool (supplied by htcondor-execute-point module)" - type = list(string) - default = [] - nullable = false - - validation { - condition = length(var.mig_id) > 0 - error_message = "At least 1 MIG containing execute points must be provided to this module" - } -} - -variable "default_mig_id" { - description = "Default MIG ID for HTCondor jobs; if unset, jobs must specify MIG id" - type = string - default = "" - nullable = false -} - -variable "enable_shielded_vm" { - type = bool - default = false - description = "Enable the Shielded VM configuration (var.shielded_instance_config)." -} - -variable "shielded_instance_config" { - description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" - type = object({ - enable_secure_boot = bool - enable_vtpm = bool - enable_integrity_monitoring = bool - }) - - default = { - enable_secure_boot = true - enable_vtpm = true - enable_integrity_monitoring = true - } -} - -variable "update_policy" { - description = "Replacement policy for Access Point Managed Instance Group (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)" - type = string - default = "OPPORTUNISTIC" - validation { - condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) - error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf deleted file mode 100644 index e64942c840..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-access-point/versions.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - random = { - source = "hashicorp/random" - version = "~> 3.6" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-access-point/v1.57.0" - } - - required_version = ">= 1.1" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md deleted file mode 100644 index dfab563a55..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/README.md +++ /dev/null @@ -1,159 +0,0 @@ -## Description - -This module provisions a highly available HTCondor central manager using a [Managed -Instance Group (MIG)][mig] with auto-healing. - -[mig]: https://cloud.google.com/compute/docs/instance-groups - -## Usage - -This module provisions an HTCondor central manager with a standard -configuration. For the node to function correctly, you must supply the input -variable described below: - -- [var.central_manager_runner](#input_central_manager_runner) - - Runner must download a POOL password / signing key and create an [IDTOKEN] - with no scopes (full authorization). - -A reference implementation is included in the Toolkit module -[htcondor-pool-secrets]. You may substitute implementations so long as they -duplicate the functionality in the references. Usage is demonstrated in the -[HTCondor example][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- -[htcondor-pool-secrets]: ../htcondor-pool-secrets/README.md -[IDTOKEN]: https://htcondor.readthedocs.io/en/latest/admin-manual/security.html#introducing-idtokens - -## Behavior of Managed Instance Group (MIG) - -A regional [MIG][mig] is used to provision the central manager, although only -1 node will ever be active at a time. By default, the node will be provisioned -in any of the zones available in that region, however, it can be constrained to -run in fewer zones (or a single zone) using [var.zones](#input_zones). - -When the configuration of the Central Manager is changed, the MIG can be -configured to [replace the VM][replacement] using a "proactive" or -"opportunistic" policy. By default, the Central Manager replacement policy is -set to proactive. In practice, this means that the Central Manager will be -replaced by Terraform when changes to the instance template / HTCondor -configuration are made. The Central Manager is safe to replace automatically as -it gathers its state information from periodic messages exchanged with the rest -of the HTCondor pool. - -This mode can be configured by setting [var.update_policy](#input_update_policy) -to either "PROACTIVE" (default) or "OPPORTUNISTIC". If set to opportunistic -replacement, the Central Manager will be replaced only when: - -- intentionally by issuing an update via Cloud Console or using gcloud (below) -- the VM becomes unhealthy or is otherwise automatically replaced (e.g. regular - Google Cloud maintenance) - -For example, to manually update all instances in a MIG: - -```text -gcloud compute instance-groups managed update-instances \ - <> --all-instances --region <> \ - --project <> --minimal-action replace -``` - -[replacement]: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups#type - -## Limiting inter-zone egress - -Because all the elements of the HTCondor pool use regional MIGs, they may be -subject to [interzone egress fees][network-pricing]. The primary traffic between -nodes of an HTCondor pool running embarrassingly parallel jobs is expected to -be limited to API traffic for job scheduling and monitoring. Please review the -[network pricing][network-pricing] documentation and determine if this cost is -a concern. If it is, use [var.zones](#input_zones) to constrain each node within -your HTCondor pool to operate within a single zone. - -[network-pricing]: https://cloud.google.com/vpc/network-pricing - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.1.0 | -| [google](#requirement\_google) | >= 3.83 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [null](#provider\_null) | >= 3.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [central\_manager\_instance\_template](#module\_central\_manager\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | -| [htcondor\_cm](#module\_htcondor\_cm) | terraform-google-modules/vm/google//modules/mig | ~> 12.1 | -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_compute_address.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | -| [google_storage_bucket_object.cm_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [null_resource.cm_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_image.htcondor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | -| [google_compute_instance.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance) | data source | -| [google_compute_region_instance_group.cm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_region_instance_group) | data source | -| [google_compute_zones.available](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_zones) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [central\_manager\_runner](#input\_central\_manager\_runner) | A list of Toolkit runners for configuring an HTCondor central manager | `list(map(string))` | `[]` | no | -| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | Service account e-mail for central manager (can be supplied by htcondor-setup module) | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `20` | no | -| [distribution\_policy\_target\_shape](#input\_distribution\_policy\_target\_shape) | Target shape for instance group managing high availability of central manager | `string` | `"ANY_SINGLE_ZONE"` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | -| [htcondor\_bucket\_name](#input\_htcondor\_bucket\_name) | Name of HTCondor configuration bucket | `string` | n/a | yes | -| [instance\_image](#input\_instance\_image) | Custom VM image with HTCondor installed using the htcondor-install module."

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | n/a | yes | -| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | -| [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor central managers | `string` | `"n2-standard-4"` | no | -| [metadata](#input\_metadata) | Metadata to add to HTCondor central managers | `map(string)` | `{}` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network in which the HTCondor central manager will be created. | `string` | `null` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | Project in which HTCondor central manager will be created | `string` | n/a | yes | -| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes by which to limit service account attached to central manager. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which the HTCondor central manager will be created. | `string` | `null` | no | -| [update\_policy](#input\_update\_policy) | Replacement policy for Central Manager ("PROACTIVE" to replace immediately or "OPPORTUNISTIC" to replace upon instance power cycle). | `string` | `"PROACTIVE"` | no | -| [zones](#input\_zones) | Zone(s) in which central manager may be created. If not supplied, will default to all zones in var.region. | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [central\_manager\_ips](#output\_central\_manager\_ips) | IP addresses of the central managers provisioned by this module | -| [central\_manager\_name](#output\_central\_manager\_name) | Name of the central managers provisioned by this module | -| [list\_instances\_command](#output\_list\_instances\_command) | Command to list central managers provisioned by this module | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml deleted file mode 100644 index 7408af6370..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/files/htcondor_configure.yml +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Configure HTCondor central manager - hosts: localhost - become: true - vars: - condor_config_root: /etc/condor - ghpc_config_file: 50-ghpc-managed - tasks: - - name: Ensure necessary variables are set - ansible.builtin.assert: - that: - - config_object is defined - - name: Remove default HTCondor configuration - ansible.builtin.file: - path: "{{ condor_config_root }}/config.d/00-htcondor-9.0.config" - state: absent - notify: - - Reload HTCondor - - name: Create Toolkit configuration file - register: config_update - changed_when: config_update.rc == 137 - failed_when: config_update.rc != 0 and config_update.rc != 137 - ansible.builtin.shell: | - set -e -o pipefail - REMOTE_HASH=$(gcloud --format="value(md5_hash)" storage hash {{ config_object }}) - - CONFIG_FILE="{{ condor_config_root }}/config.d/{{ ghpc_config_file }}" - if [ -f "${CONFIG_FILE}" ]; then - LOCAL_HASH=$(gcloud --format="value(md5_hash)" storage hash "${CONFIG_FILE}") - else - LOCAL_HASH="INVALID-HASH" - fi - - if [ "${REMOTE_HASH}" != "${LOCAL_HASH}" ]; then - gcloud storage cp {{ config_object }} "${CONFIG_FILE}" - chmod 0644 "${CONFIG_FILE}" - exit 137 - fi - args: - executable: /bin/bash - notify: - - Reload HTCondor - handlers: - - name: Reload HTCondor - ansible.builtin.service: - name: condor - state: reloaded - post_tasks: - - name: Start HTCondor - ansible.builtin.service: - name: condor - state: started - enabled: true - - name: Inform users - changed_when: false - ansible.builtin.shell: | - set -e -o pipefail - wall "******* HTCondor system configuration complete ********" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf deleted file mode 100644 index d288a91144..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/main.tf +++ /dev/null @@ -1,226 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "htcondor-central-manager", ghpc_role = "scheduler" }) -} - -locals { - network_storage_metadata = var.network_storage == null ? {} : { network_storage = jsonencode(var.network_storage) } - oslogin_api_values = { - "DISABLE" = "FALSE" - "ENABLE" = "TRUE" - } - enable_oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - metadata = merge( - local.network_storage_metadata, - local.enable_oslogin_metadata, - local.disable_automatic_updates_metadata, - var.metadata - ) - - name_prefix = "${var.deployment_name}-cm" - - cm_config = templatefile("${path.module}/templates/condor_config.tftpl", {}) - - cm_object = "gs://${var.htcondor_bucket_name}/${google_storage_bucket_object.cm_config.output_name}" - schedd_runner = { - type = "ansible-local" - content = file("${path.module}/files/htcondor_configure.yml") - destination = "htcondor_configure.yml" - args = join(" ", [ - "-e config_object=${local.cm_object}", - ]) - } - - native_fstype = [] - startup_script_network_storage = [ - for ns in var.network_storage : - ns if !contains(local.native_fstype, ns.fs_type) - ] - storage_client_install_runners = [ - for ns in local.startup_script_network_storage : - ns.client_install_runner if ns.client_install_runner != null - ] - mount_runners = [ - for ns in local.startup_script_network_storage : - ns.mount_runner if ns.mount_runner != null - ] - - all_runners = concat( - local.storage_client_install_runners, - local.mount_runners, - var.central_manager_runner, - [local.schedd_runner] - ) - - central_manager_ips = google_compute_address.cm.address - central_manager_name = data.google_compute_instance.cm.name - - list_instances_command = "gcloud compute instance-groups list-instances ${data.google_compute_region_instance_group.cm.name} --region ${var.region} --project ${var.project_id}" - - zones = coalescelist(var.zones, data.google_compute_zones.available.names) -} - -data "google_compute_image" "htcondor" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = var.instance_image.project - - lifecycle { - postcondition { - condition = self.disk_size_gb <= var.disk_size_gb - error_message = "var.disk_size_gb must be set to at least the size of the image (${self.disk_size_gb})" - } - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} - -data "google_compute_zones" "available" { - project = var.project_id - region = var.region -} - -data "google_compute_region_instance_group" "cm" { - self_link = module.htcondor_cm.self_link - lifecycle { - postcondition { - condition = length(self.instances) == 1 - error_message = "There should only be 1 central manager found" - } - } -} - -data "google_compute_instance" "cm" { - self_link = data.google_compute_region_instance_group.cm.instances[0].instance -} - -resource "null_resource" "cm_config" { - triggers = { - config = local.cm_config - } -} - -resource "google_storage_bucket_object" "cm_config" { - name = "${local.name_prefix}-config-${substr(md5(null_resource.cm_config.id), 0, 4)}" - content = local.cm_config - bucket = var.htcondor_bucket_name -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - project_id = var.project_id - region = var.region - labels = local.labels - deployment_name = var.deployment_name - - runners = local.all_runners -} - -resource "google_compute_address" "cm" { - project = var.project_id - name = local.name_prefix - region = var.region - subnetwork = var.subnetwork_self_link - address_type = "INTERNAL" - purpose = "GCE_ENDPOINT" -} - -module "central_manager_instance_template" { - source = "terraform-google-modules/vm/google//modules/instance_template" - version = "~> 12.1" - - name_prefix = local.name_prefix - project_id = var.project_id - network = var.network_self_link - subnetwork = var.subnetwork_self_link - service_account = { - email = var.central_manager_service_account_email - scopes = var.service_account_scopes - } - labels = local.labels - - machine_type = var.machine_type - disk_size_gb = var.disk_size_gb - preemptible = false - startup_script = module.startup_script.startup_script - metadata = local.metadata - source_image = data.google_compute_image.htcondor.self_link - - # secure boot - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config - - network_ip = google_compute_address.cm.id -} - -module "htcondor_cm" { - source = "terraform-google-modules/vm/google//modules/mig" - version = "~> 12.1" - - project_id = var.project_id - region = var.region - distribution_policy_target_shape = var.distribution_policy_target_shape - distribution_policy_zones = local.zones - target_size = 1 - hostname = local.name_prefix - instance_template = module.central_manager_instance_template.self_link - - health_check_name = "health-${local.name_prefix}" - health_check = { - type = "tcp" - initial_delay_sec = 600 - check_interval_sec = 20 - healthy_threshold = 2 - timeout_sec = 8 - unhealthy_threshold = 3 - response = "" - proxy_header = "NONE" - port = 9618 - request = "" - request_path = "" - host = "" - enable_logging = true - } - - update_policy = [{ - instance_redistribution_type = "NONE" - replacement_method = "RECREATE" # preserves hostnames (necessary for PROACTIVE replacement) - max_surge_fixed = 0 # must be 0 to preserve hostnames - max_unavailable_fixed = length(local.zones) - max_surge_percent = null - max_unavailable_percent = null - min_ready_sec = 300 - minimal_action = "REPLACE" - type = var.update_policy - }] - - # the timeouts below are default for resource - wait_for_instances = true - mig_timeouts = { - create = "15m" - delete = "15m" - update = "15m" - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml deleted file mode 100644 index 3a78f9a46b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - storage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf deleted file mode 100644 index a6272e7ca2..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "list_instances_command" { - description = "Command to list central managers provisioned by this module" - value = local.list_instances_command -} - -output "central_manager_ips" { - description = "IP addresses of the central managers provisioned by this module" - value = local.central_manager_ips -} - -output "central_manager_name" { - description = "Name of the central managers provisioned by this module" - value = local.central_manager_name -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl deleted file mode 100644 index 5b9676457e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/templates/condor_config.tftpl +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# this file is managed by the Cluster Toolkit; do not edit it manually -# override settings with a higher priority (last lexically) named file -# https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html?#ordered-evaluation-to-set-the-configuration - -use role:get_htcondor_central_manager -CONDOR_HOST = $(IPV4_ADDRESS) - -# Central Manager configuration settings -# https://htcondor.readthedocs.io/en/23.0/admin-manual/configuration-macros.html#condor-collector-configuration-file-entries -# https://htcondor.readthedocs.io/en/23.0/admin-manual/configuration-macros.html#condor-negotiator-configuration-file-entries -# set classad lifetime (expiration) to ~5x the update interval for all daemons -# defaults to 900s -CLASSAD_LIFETIME = 180 -COLLECTOR_UPDATE_INTERVAL = 30 -NEGOTIATOR_UPDATE_INTERVAL = 30 -NEGOTIATOR_DEPTH_FIRST = True -NEGOTIATOR_UPDATE_AFTER_CYCLE = True diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf deleted file mode 100644 index 7f85861c3f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/variables.tf +++ /dev/null @@ -1,192 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which HTCondor central manager will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "labels" { - description = "Labels to add to resources. List key, value pairs." - type = map(string) -} - -variable "region" { - description = "Default region for creating resources" - type = string -} - -variable "zones" { - description = "Zone(s) in which central manager may be created. If not supplied, will default to all zones in var.region." - type = list(string) - default = [] - nullable = false -} - -variable "distribution_policy_target_shape" { - description = "Target shape for instance group managing high availability of central manager" - type = string - default = "ANY_SINGLE_ZONE" -} - -variable "network_self_link" { - description = "The self link of the network in which the HTCondor central manager will be created." - type = string - default = null -} - -variable "central_manager_service_account_email" { - description = "Service account e-mail for central manager (can be supplied by htcondor-setup module)" - type = string -} - -variable "service_account_scopes" { - description = "Scopes by which to limit service account attached to central manager." - type = set(string) - default = [ - "https://www.googleapis.com/auth/cloud-platform", - ] -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured" - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "disk_size_gb" { - description = "Boot disk size in GB" - type = number - default = 20 - nullable = false -} - -variable "metadata" { - description = "Metadata to add to HTCondor central managers" - type = map(string) - default = {} -} - -variable "enable_oslogin" { - description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." - type = string - default = "ENABLE" - nullable = false - validation { - condition = contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) - error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." - } -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork in which the HTCondor central manager will be created." - type = string - default = null -} - -variable "instance_image" { - description = <<-EOD - Custom VM image with HTCondor installed using the htcondor-install module." - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "machine_type" { - description = "Machine type to use for HTCondor central managers" - type = string - default = "n2-standard-4" -} - -variable "central_manager_runner" { - description = "A list of Toolkit runners for configuring an HTCondor central manager" - type = list(map(string)) - default = [] -} - -variable "htcondor_bucket_name" { - description = "Name of HTCondor configuration bucket" - type = string -} - -variable "enable_shielded_vm" { - type = bool - default = false - description = "Enable the Shielded VM configuration (var.shielded_instance_config)." -} - -variable "shielded_instance_config" { - description = "Shielded VM configuration for the instance (must set var.enabled_shielded_vm)" - type = object({ - enable_secure_boot = bool - enable_vtpm = bool - enable_integrity_monitoring = bool - }) - - default = { - enable_secure_boot = true - enable_vtpm = true - enable_integrity_monitoring = true - } -} - -variable "update_policy" { - description = "Replacement policy for Central Manager (\"PROACTIVE\" to replace immediately or \"OPPORTUNISTIC\" to replace upon instance power cycle)." - type = string - default = "PROACTIVE" - validation { - condition = contains(["PROACTIVE", "OPPORTUNISTIC"], var.update_policy) - error_message = "Allowed string values for var.update_policy are \"PROACTIVE\" or \"OPPORTUNISTIC\"." - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf deleted file mode 100644 index 24051328ff..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-central-manager/versions.tf +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-central-manager/v1.57.0" - } - - required_version = ">= 1.1.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md deleted file mode 100644 index 7158e7bac6..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/README.md +++ /dev/null @@ -1,172 +0,0 @@ -## Description - -This module is responsible for the following actions: - -- store an HTCondor Pool password in Google Cloud Secret Manager - - will generate a new password if one is not supplied -- create a secret in Google Cloud Secret Manager in which the HTCondor central - manager can place IDTOKENs (JWT Authorizations) for execute points to download -- create a Toolkit runner for the central manager - - download the POOL password / signing key - - create a local IDTOKEN for itself - - upload the execute point IDTOKEN secret -- create a Toolkit runner for access points - - download the POOL password / signing key - - create a local IDTOKEN for itself -- create a Toolkit runner for execute points - - Fetch the IDTOKEN secret generated by the central manager - -It is expected to be used with the [htcondor-install] and -[htcondor-execute-point] modules. - -[hpcvmimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm -[htcondor-install]: ../../scripts/htcondor-setup/README.md -[htcondor-execute-point]: ../../compute/htcondor-execute-point/README.md - -[htcrole]: https://htcondor.readthedocs.io/en/latest/getting-htcondor/admin-quick-start.html#what-get-htcondor-does-to-configure-a-role - -### Example - -The following code snippet uses this module to create a startup script that -installs HTCondor software and configures an HTCondor Central Manager. A full -example can be found in the [examples README][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- - -```yaml -- id: network1 - source: modules/network/pre-existing-vpc - -- id: htcondor_install - source: community/modules/scripts/htcondor-install - -- id: htcondor_setup - source: community/modules/scheduler/htcondor-setup - use: - - network1 - -- id: htcondor_secrets - source: community/modules/scheduler/htcondor-pool-secrets - use: - - htcondor_setup - - - id: htcondor_startup_central_manager - source: modules/scripts/startup-script - settings: - runners: - - $(htcondor_install.install_htcondor_runner) - - $(htcondor_secrets.central_manager_runner) - - $(htcondor_setup.central_manager_runner) - -- id: htcondor_cm - source: modules/compute/vm-instance - use: - - network1 - - htcondor_startup_central_manager - settings: - name_prefix: cm0 - machine_type: c2-standard-4 - disable_public_ips: true - service_account: - email: $(htcondor_setup.central_manager_service_account) - scopes: - - cloud-platform - network_interfaces: - - network: null - subnetwork: $(network1.subnetwork_self_link) - subnetwork_project: $(vars.project_id) - network_ip: $(htcondor_setup.central_manager_internal_ip) - stack_type: null - access_config: [] - ipv6_access_config: [] - alias_ip_range: [] - nic_type: VIRTIO_NET - queue_count: null - outputs: - - internal_ip -``` - -## Support - -HTCondor is maintained by the [Center for High Throughput Computing][chtc] at -the University of Wisconsin-Madison. Support for HTCondor is available via: - -- [Discussion lists](https://htcondor.org/mail-lists/) -- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) -- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) - -[chtc]: https://chtc.cs.wisc.edu/ - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [google](#requirement\_google) | >= 4.84 | -| [random](#requirement\_random) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.84 | -| [random](#provider\_random) | >= 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_secret_manager_secret.execute_point_idtoken](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | -| [google_secret_manager_secret.pool_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | -| [google_secret_manager_secret_iam_member.access_point](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_iam_member.central_manager_idtoken](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_iam_member.central_manager_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_iam_member.execute_point](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_version.pool_password](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | -| [random_password.pool](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | -| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | HTCondor access point service account e-mail | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | -| [pool\_password](#input\_pool\_password) | HTCondor Pool Password | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | -| [trust\_domain](#input\_trust\_domain) | Trust domain for HTCondor pool (if not supplied, will be set based on project\_id) | `string` | `""` | no | -| [user\_managed\_replication](#input\_user\_managed\_replication) | Replication parameters that will be used for defined secrets | 
list(object({
    location     = string
    kms_key_name = optional(string)
  }))
| `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [access\_point\_runner](#output\_access\_point\_runner) | Toolkit Runner to download pool secrets to an HTCondor access point | -| [central\_manager\_runner](#output\_central\_manager\_runner) | Toolkit Runner to download pool secrets to an HTCondor central manager | -| [execute\_point\_runner](#output\_execute\_point\_runner) | Toolkit Runner to download pool secrets to an HTCondor execute point | -| [pool\_password\_secret\_id](#output\_pool\_password\_secret\_id) | Google Cloud Secret Manager ID containing HTCondor Pool Password | -| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | PowerShell script to download pool secrets to an HTCondor execute point | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml deleted file mode 100644 index 538c809c2a..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/files/htcondor_secrets.yml +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Configure HTCondor Secrets - hosts: localhost - become: true - vars: - condor_config_root: /etc/condor - tasks: - - name: Ensure necessary variables are set - ansible.builtin.assert: - that: - - htcondor_role is defined - - password_id is defined - - trust_domain is defined - - name: Set Pool Trust Domain - ansible.builtin.copy: - dest: "{{ condor_config_root }}/config.d/51-ghpc-trust-domain" - mode: 0644 - content: | - # these lines must appear AFTER any "use role:" settings - UID_DOMAIN = {{ trust_domain }} - TRUST_DOMAIN = {{ trust_domain }} - - name: Get HTCondor Pool password (token signing key) - when: htcondor_role != 'get_htcondor_execute' - ansible.builtin.shell: | - set -e -o pipefail +o history - POOL_PASSWORD=$(gcloud secrets versions access latest --secret={{ password_id }}) - echo -n "$POOL_PASSWORD" | sh -c "condor_store_cred add -c -i -" - args: - creates: "{{ condor_config_root }}/passwords.d/POOL" - executable: /bin/bash - - name: Configure HTCondor Central Manager - when: htcondor_role == 'get_htcondor_central_manager' - block: - - name: Create IDTOKEN for Central Manager - ansible.builtin.shell: | - umask 0077 - condor_token_create -identity condor@{{ trust_domain }} \ - -token condor@{{ trust_domain }} - args: - creates: "{{ condor_config_root }}/tokens.d/condor@{{ trust_domain }}" - - name: Create IDTOKEN secret for Execute Points - when: xp_idtoken_secret_id | length > 0 - changed_when: true - ansible.builtin.shell: | - umask 0077 - TMPFILE=$(mktemp) - condor_token_create -authz READ -authz ADVERTISE_MASTER \ - -authz ADVERTISE_STARTD -identity condor@{{ trust_domain }} > "$TMPFILE" - gcloud secrets versions add --data-file "$TMPFILE" {{ xp_idtoken_secret_id }} - rm -f "$TMPFILE" - - name: Configure HTCondor SchedD - when: htcondor_role == 'get_htcondor_submit' - block: - - name: Create IDTOKEN to advertise access point - ansible.builtin.shell: | - umask 0077 - # DAEMON authorization can likely be removed in future when scopes - # needed to trigger a negotiation cycle are changed. Suggest review - # https://opensciencegrid.atlassian.net/jira/software/c/projects/HTCONDOR/issues/?filter=allissues - condor_token_create -authz READ -authz ADVERTISE_MASTER \ - -authz ADVERTISE_SCHEDD -authz DAEMON -identity condor@{{ trust_domain }} \ - -token condor@{{ trust_domain }} - args: - creates: "{{ condor_config_root }}/tokens.d/condor@{{ trust_domain }}" - - name: Configure HTCondor StartD - when: htcondor_role == 'get_htcondor_execute' - block: - - name: Create SystemD override directory for HTCondor Execute Point - ansible.builtin.file: - path: /etc/systemd/system/condor.service.d - state: directory - owner: root - group: root - mode: 0755 - - name: Fetch IDTOKEN to advertise execute point - ansible.builtin.copy: - dest: "/etc/systemd/system/condor.service.d/htcondor-token-fetcher.conf" - mode: 0644 - content: | - [Service] - ExecStartPre=gcloud secrets versions access latest --secret {{ xp_idtoken_secret_id }} \ - --out-file {{ condor_config_root }}/tokens.d/condor@{{ trust_domain }} - notify: - - Reload SystemD - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf deleted file mode 100644 index 1a7c761760..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/main.tf +++ /dev/null @@ -1,168 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "htcondor-pool-secrets", ghpc_role = "scheduler" }) -} - -locals { - pool_password = coalesce(var.pool_password, random_password.pool.result) - auto = length(var.user_managed_replication) == 0 ? "" : "-user" - access_point_service_account_iam_email = "serviceAccount:${var.access_point_service_account_email}" - central_manager_service_account_iam_email = "serviceAccount:${var.central_manager_service_account_email}" - execute_point_service_account_iam_email = "serviceAccount:${var.execute_point_service_account_email}" - - trust_domain = coalesce(var.trust_domain, "c.${var.project_id}.internal") - - runner_cm = { - "type" = "ansible-local" - "content" = file("${path.module}/files/htcondor_secrets.yml") - "destination" = "htcondor_secrets.yml" - "args" = join(" ", [ - "-e htcondor_role=get_htcondor_central_manager", - "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", - "-e xp_idtoken_secret_id=${google_secret_manager_secret.execute_point_idtoken.secret_id}", - "-e trust_domain=${local.trust_domain}", - ]) - } - - runner_access = { - "type" = "ansible-local" - "content" = file("${path.module}/files/htcondor_secrets.yml") - "destination" = "htcondor_secrets.yml" - "args" = join(" ", [ - "-e htcondor_role=get_htcondor_submit", - "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", - "-e trust_domain=${local.trust_domain}", - ]) - } - - runner_execute = { - "type" = "ansible-local" - "content" = file("${path.module}/files/htcondor_secrets.yml") - "destination" = "htcondor_secrets.yml" - "args" = join(" ", [ - "-e htcondor_role=get_htcondor_execute", - "-e password_id=${google_secret_manager_secret.pool_password.secret_id}", - "-e xp_idtoken_secret_id=${google_secret_manager_secret.execute_point_idtoken.secret_id}", - "-e trust_domain=${local.trust_domain}", - ]) - } - windows_startup_ps1 = templatefile( - "${path.module}/templates/fetch-idtoken.ps1.tftpl", - { - trust_domain = local.trust_domain, - xp_idtoken_secret_id = google_secret_manager_secret.execute_point_idtoken.secret_id, - } - ) -} - -resource "random_password" "pool" { - length = 24 - special = true - override_special = "_-#=." -} - -resource "google_secret_manager_secret" "pool_password" { - secret_id = "${var.deployment_name}-pool-password${local.auto}" - - labels = local.labels - - replication { - dynamic "auto" { - for_each = length(var.user_managed_replication) == 0 ? [1] : [] - content {} - } - dynamic "user_managed" { - for_each = length(var.user_managed_replication) == 0 ? [] : [1] - content { - dynamic "replicas" { - for_each = var.user_managed_replication - content { - location = replicas.value.location - dynamic "customer_managed_encryption" { - for_each = compact([replicas.value.kms_key_name]) - content { - kms_key_name = customer_managed_encryption.value - } - } - } - } - } - } - } -} - -resource "google_secret_manager_secret_version" "pool_password" { - secret = google_secret_manager_secret.pool_password.id - secret_data = local.pool_password -} - -# this secret will be populated by the Central Manager -resource "google_secret_manager_secret" "execute_point_idtoken" { - secret_id = "${var.deployment_name}-execute-point-idtoken${local.auto}" - - labels = local.labels - - replication { - dynamic "auto" { - for_each = length(var.user_managed_replication) == 0 ? [1] : [] - content {} - } - dynamic "user_managed" { - for_each = length(var.user_managed_replication) == 0 ? [] : [1] - content { - dynamic "replicas" { - for_each = var.user_managed_replication - content { - location = replicas.value.location - dynamic "customer_managed_encryption" { - for_each = compact([replicas.value.kms_key_name]) - content { - kms_key_name = customer_managed_encryption.value - } - } - } - } - } - } - } -} - -resource "google_secret_manager_secret_iam_member" "central_manager_password" { - secret_id = google_secret_manager_secret.pool_password.id - role = "roles/secretmanager.secretAccessor" - member = local.central_manager_service_account_iam_email -} - -resource "google_secret_manager_secret_iam_member" "central_manager_idtoken" { - secret_id = google_secret_manager_secret.execute_point_idtoken.id - role = "roles/secretmanager.secretVersionManager" - member = local.central_manager_service_account_iam_email -} - -resource "google_secret_manager_secret_iam_member" "access_point" { - secret_id = google_secret_manager_secret.pool_password.id - role = "roles/secretmanager.secretAccessor" - member = local.access_point_service_account_iam_email -} - -resource "google_secret_manager_secret_iam_member" "execute_point" { - secret_id = google_secret_manager_secret.execute_point_idtoken.id - role = "roles/secretmanager.secretAccessor" - member = local.execute_point_service_account_iam_email -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml deleted file mode 100644 index 4b0bdbd616..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - iam.googleapis.com - - secretmanager.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf deleted file mode 100644 index 81c4986b16..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/outputs.tf +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "pool_password_secret_id" { - description = "Google Cloud Secret Manager ID containing HTCondor Pool Password" - value = google_secret_manager_secret.pool_password.secret_id - sensitive = true -} - -output "central_manager_runner" { - description = "Toolkit Runner to download pool secrets to an HTCondor central manager" - value = local.runner_cm - depends_on = [ - google_secret_manager_secret_version.pool_password - ] -} - -output "access_point_runner" { - description = "Toolkit Runner to download pool secrets to an HTCondor access point" - value = local.runner_access - depends_on = [ - google_secret_manager_secret_version.pool_password - ] -} - -output "execute_point_runner" { - description = "Toolkit Runner to download pool secrets to an HTCondor execute point" - value = local.runner_execute - depends_on = [ - google_secret_manager_secret_version.pool_password - ] -} - -output "windows_startup_ps1" { - description = "PowerShell script to download pool secrets to an HTCondor execute point" - value = local.windows_startup_ps1 -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl deleted file mode 100644 index 04c96291ee..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/templates/fetch-idtoken.ps1.tftpl +++ /dev/null @@ -1,26 +0,0 @@ -Set-StrictMode -Version latest -$ErrorActionPreference = 'Stop' - -$config_dir = 'C:\Condor\config' -if(!(test-path -PathType container -Path $config_dir)) -{ - New-Item -ItemType Directory -Path $config_dir -} -$config_file = "$config_dir\51-ghpc-trust-domain" - -$config_string = @' -# these lines must appear AFTER any "use role:" settings -UID_DOMAIN = ${trust_domain} -TRUST_DOMAIN = ${trust_domain} -'@ - -Set-Content -Path "$config_file" -Value "$config_string" - -# obtain IDTOKEN for authentication by StartD to Central Manager -gcloud secrets versions access latest --secret ${xp_idtoken_secret_id} ` - --out-file C:\condor\tokens.d\condor@${trust_domain} - -if ($LASTEXITCODE -ne 0) -{ - throw "Could not download HTCondor IDTOKEN; exiting startup script" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf deleted file mode 100644 index 22ef3644e8..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/variables.tf +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which HTCondor pool will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "labels" { - description = "Labels to add to resources. List key, value pairs." - type = map(string) -} - -variable "access_point_service_account_email" { - description = "HTCondor access point service account e-mail" - type = string -} - -variable "central_manager_service_account_email" { - description = "HTCondor access point service account e-mail" - type = string -} - -variable "execute_point_service_account_email" { - description = "HTCondor access point service account e-mail" - type = string -} - -variable "pool_password" { - description = "HTCondor Pool Password" - type = string - sensitive = true - default = null -} - -variable "trust_domain" { - description = "Trust domain for HTCondor pool (if not supplied, will be set based on project_id)" - type = string - default = "" -} - -variable "user_managed_replication" { - type = list(object({ - location = string - kms_key_name = optional(string) - })) - description = "Replication parameters that will be used for defined secrets" - default = [] -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf deleted file mode 100644 index 87a806a0ea..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-pool-secrets/versions.tf +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.84" - } - random = { - source = "hashicorp/random" - version = ">= 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-pool-secrets/v1.57.0" - } - - required_version = ">= 1.3.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md deleted file mode 100644 index 5a403c0a38..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/README.md +++ /dev/null @@ -1,128 +0,0 @@ -## Description - -This module creates the service accounts for use by the primary elements of an -[HTCondor pool][pool]: - -- Central Managers -- Access Points -- Execute Points - -Each service account is assigned common roles necessary for the VM to function -properly. In particular, nearly every VM requires the ability to read from Cloud -Storage buckets and write Cloud Logging entries. These roles are configurable -as described below. - -[pool]: https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-admin-manual.html#the-different-roles-a-machine-can-play - -### Example - -The following code snippet uses this module to create a startup script that -installs HTCondor software and configures an HTCondor Central Manager. A full -example can be found in the [examples README][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- - -```yaml -- id: network1 - source: modules/network/pre-existing-vpc - -- id: htcondor_install - source: community/modules/scripts/htcondor-install - -- id: htcondor_service_accounts - source: community/modules/scheduler/htcondor-service-accounts - -- id: htcondor_setup - source: community/modules/scheduler/htcondor-setup - use: - - network1 - - htcondor_service_accounts - -- id: htcondor_secrets - source: community/modules/scheduler/htcondor-pool-secrets - use: - - htcondor_service_accounts - -- id: htcondor_cm - source: community/modules/scheduler/htcondor-central-manager - use: - - network1 - - htcondor_secrets - - htcondor_service_accounts - - htcondor_setup - settings: - instance_image: - project: $(vars.project_id) - family: $(vars.new_image_family) - outputs: - - central_manager_name -``` - -## Support - -HTCondor is maintained by the [Center for High Throughput Computing][chtc] at -the University of Wisconsin-Madison. Support for HTCondor is available via: - -- [Discussion lists](https://htcondor.org/mail-lists/) -- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) -- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) - -[chtc]: https://chtc.cs.wisc.edu/ - -## License - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [access\_point\_service\_account](#module\_access\_point\_service\_account) | ../../../../community/modules/project/service-account | n/a | -| [central\_manager\_service\_account](#module\_central\_manager\_service\_account) | ../../../../community/modules/project/service-account | n/a | -| [execute\_point\_service\_account](#module\_execute\_point\_service\_account) | ../../../../community/modules/project/service-account | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_point\_roles](#input\_access\_point\_roles) | Project-wide roles for HTCondor Access Point service account | `list(string)` | 
[
  "compute.instanceAdmin.v1",
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | -| [central\_manager\_roles](#input\_central\_manager\_roles) | Project-wide roles for HTCondor Central Manager service account | `list(string)` | 
[
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [execute\_point\_roles](#input\_execute\_point\_roles) | Project-wide roles for HTCondor Execute Point service account | `list(string)` | 
[
  "monitoring.metricWriter",
  "logging.logWriter",
  "storage.objectViewer"
]
| no | -| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [access\_point\_service\_account\_email](#output\_access\_point\_service\_account\_email) | HTCondor Access Point Service Account (e-mail format) | -| [central\_manager\_service\_account\_email](#output\_central\_manager\_service\_account\_email) | HTCondor Central Manager Service Account (e-mail format) | -| [execute\_point\_service\_account\_email](#output\_execute\_point\_service\_account\_email) | HTCondor Execute Point Service Account (e-mail format) | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf deleted file mode 100644 index 9d97b18642..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/main.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# NB: the community/modules/project/service-account module will not output the -# service account e-mail address until all IAM bindings have been created; if -# underlying implementation changes, this module should declare explicit -# depends_on the IAM bindings to prevent race conditions for services that -# require them - -module "access_point_service_account" { - source = "../../../../community/modules/project/service-account" - - project_id = var.project_id - display_name = "HTCondor Access Point" - deployment_name = var.deployment_name - name = "access" - project_roles = var.access_point_roles -} - -module "execute_point_service_account" { - source = "../../../../community/modules/project/service-account" - - project_id = var.project_id - display_name = "HTCondor Execute Point" - deployment_name = var.deployment_name - name = "execute" - project_roles = var.execute_point_roles -} - -module "central_manager_service_account" { - source = "../../../../community/modules/project/service-account" - - project_id = var.project_id - display_name = "HTCondor Central Manager" - deployment_name = var.deployment_name - name = "cm" - project_roles = var.central_manager_roles -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml deleted file mode 100644 index c4dcdffdf4..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - iam.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf deleted file mode 100644 index 28f3a79457..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "access_point_service_account_email" { - description = "HTCondor Access Point Service Account (e-mail format)" - value = module.access_point_service_account.service_account_email -} - -output "central_manager_service_account_email" { - description = "HTCondor Central Manager Service Account (e-mail format)" - value = module.central_manager_service_account.service_account_email -} - -output "execute_point_service_account_email" { - description = "HTCondor Execute Point Service Account (e-mail format)" - value = module.execute_point_service_account.service_account_email -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf deleted file mode 100644 index ee186e0971..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/variables.tf +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which HTCondor pool will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "access_point_roles" { - description = "Project-wide roles for HTCondor Access Point service account" - type = list(string) - default = [ - "compute.instanceAdmin.v1", - "monitoring.metricWriter", - "logging.logWriter", - "storage.objectViewer", - ] -} - -variable "central_manager_roles" { - description = "Project-wide roles for HTCondor Central Manager service account" - type = list(string) - default = [ - "monitoring.metricWriter", - "logging.logWriter", - "storage.objectViewer", - ] -} - -variable "execute_point_roles" { - description = "Project-wide roles for HTCondor Execute Point service account" - type = list(string) - default = [ - "monitoring.metricWriter", - "logging.logWriter", - "storage.objectViewer", - ] -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf deleted file mode 100644 index 79b6fbde47..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-service-accounts/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.13.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/README.md deleted file mode 100644 index 9c46f0a3c8..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/README.md +++ /dev/null @@ -1,118 +0,0 @@ -## Description - -This module creates a bucket in which to store HTCondor configurations and -a firewall rule that allows Managed Instance Group health checks to probe the -health of HTCondor VMs. - -### Example - -The following code snippet uses this module to create a startup script that -installs HTCondor software and configures an HTCondor Central Manager. A full -example can be found in the [examples README][htc-example]. - -[htc-example]: ../../../../examples/README.md#htc-htcondoryaml-- - -```yaml -- id: network1 - source: modules/network/pre-existing-vpc - -- id: htcondor_install - source: community/modules/scripts/htcondor-install - -- id: htcondor_service_accounts - source: community/modules/scheduler/htcondor-service-accounts - -- id: htcondor_setup - source: community/modules/scheduler/htcondor-setup - use: - - network1 - - htcondor_service_accounts - -- id: htcondor_secrets - source: community/modules/scheduler/htcondor-pool-secrets - use: - - htcondor_service_accounts - -- id: htcondor_cm - source: community/modules/scheduler/htcondor-central-manager - use: - - network1 - - htcondor_secrets - - htcondor_service_accounts - - htcondor_setup - settings: - instance_image: - project: $(vars.project_id) - family: $(vars.new_image_family) - outputs: - - central_manager_name -``` - -## Support - -HTCondor is maintained by the [Center for High Throughput Computing][chtc] at -the University of Wisconsin-Madison. Support for HTCondor is available via: - -- [Discussion lists](https://htcondor.org/mail-lists/) -- [HTCondor on GitHub](https://github.com/htcondor/htcondor/) -- [HTCondor manual](https://htcondor.readthedocs.io/en/latest/) - -[chtc]: https://chtc.cs.wisc.edu/ - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [health\_check\_firewall\_rule](#module\_health\_check\_firewall\_rule) | ../../../../modules/network/firewall-rules | n/a | -| [htcondor\_bucket](#module\_htcondor\_bucket) | ../../../../community/modules/file-system/cloud-storage-bucket | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_point\_service\_account\_email](#input\_access\_point\_service\_account\_email) | Service account e-mail for HTCondor Access Point | `string` | n/a | yes | -| [central\_manager\_service\_account\_email](#input\_central\_manager\_service\_account\_email) | Service account e-mail for HTCondor Central Manager | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [execute\_point\_service\_account\_email](#input\_execute\_point\_service\_account\_email) | Service account e-mail for HTCondor Execute Points | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to resources. List key, value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which HTCondor pool will be created | `string` | n/a | yes | -| [region](#input\_region) | Default region for creating resources | `string` | n/a | yes | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork in which Central Managers will be placed. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [htcondor\_bucket\_name](#output\_htcondor\_bucket\_name) | Name of the HTCondor configuration bucket | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf deleted file mode 100644 index ad471fddad..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/main.tf +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "htcondor-setup", ghpc_role = "scheduler" }) -} - -locals { - service_account_iam_email = [ - "serviceAccount:${var.access_point_service_account_email}", - "serviceAccount:${var.central_manager_service_account_email}", - "serviceAccount:${var.execute_point_service_account_email}", - ] - service_account_email = [ - var.access_point_service_account_email, - var.central_manager_service_account_email, - var.execute_point_service_account_email, - ] -} - -module "health_check_firewall_rule" { - source = "../../../../modules/network/firewall-rules" - - subnetwork_self_link = var.subnetwork_self_link - - ingress_rules = [{ - name = "allow-health-check-${var.deployment_name}" - description = "Allow Managed Instance Group Health Checks for HTCondor VMs" - direction = "INGRESS" - source_ranges = [ - "130.211.0.0/22", - "35.191.0.0/16", - ] - target_service_accounts = local.service_account_email - allow = [{ - protocol = "tcp" - ports = ["9618"] - }] - }] -} - -module "htcondor_bucket" { - source = "../../../../community/modules/file-system/cloud-storage-bucket" - - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - name_prefix = "${var.deployment_name}-htcondor-config" - random_suffix = true - labels = local.labels - viewers = local.service_account_iam_email - - use_deployment_name_in_bucket_name = false -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml deleted file mode 100644 index 7b4918b962..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - iam.googleapis.com - - storage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf deleted file mode 100644 index a44223faee..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/outputs.tf +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "htcondor_bucket_name" { - description = "Name of the HTCondor configuration bucket" - value = module.htcondor_bucket.gcs_bucket_name - - # ensure that all IAM bindings to the bucket and firewall rules are active - # before this modules output is allowed to propagate - depends_on = [ - module.htcondor_bucket, - module.health_check_firewall_rule - ] -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf deleted file mode 100644 index 147a2ca88d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/variables.tf +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which HTCondor pool will be created" - type = string -} - -variable "deployment_name" { - description = "Cluster Toolkit deployment name. HTCondor cloud resource names will include this value." - type = string -} - -variable "labels" { - description = "Labels to add to resources. List key, value pairs." - type = map(string) -} - -variable "region" { - description = "Default region for creating resources" - type = string -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork in which Central Managers will be placed." - type = string -} - -variable "access_point_service_account_email" { - description = "Service account e-mail for HTCondor Access Point" - type = string -} - -variable "central_manager_service_account_email" { - description = "Service account e-mail for HTCondor Central Manager" - type = string -} - -variable "execute_point_service_account_email" { - description = "Service account e-mail for HTCondor Execute Points" - type = string -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf deleted file mode 100644 index 79b6fbde47..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/htcondor-setup/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.13.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md deleted file mode 100644 index 7ddb6058c4..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md +++ /dev/null @@ -1,405 +0,0 @@ -## Description - -This module creates a slurm controller node via the internal -[slurm\_instance\_template] module. - -More information about Slurm On GCP can be found at the -[project's GitHub page][slurm-gcp] and in the -[Slurm on Google Cloud User Guide][slurm-ug]. - -The [user guide][slurm-ug] provides detailed instructions on customizing and -enhancing the Slurm on GCP cluster as well as recommendations on configuring the -controller for optimal performance at different scales. - -[slurm\_instance\_template]: /community/modules/internal/slurm-gcp/instance_template/README.md -[slurm-ug]: https://goo.gle/slurm-gcp-user-guide. -[enable\_cleanup\_compute]: #input\_enable\_cleanup\_compute -[enable\_cleanup\_subscriptions]: #input\_enable\_cleanup\_subscriptions -[enable\_reconfigure]: #input\_enable\_reconfigure - -### Example - -```yaml -- id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - use: - - network - - homefs - - compute_partition - settings: - machine_type: c2-standard-8 -``` - -This creates a controller node with the following attributes: - -* connected to the primary subnetwork of `network` -* the filesystem with the ID `homefs` (defined elsewhere in the blueprint) - mounted -* One partition with the ID `compute_partition` (defined elsewhere in the - blueprint) -* machine type upgraded from the default `c2-standard-4` to `c2-standard-8` - -### Live Cluster Reconfiguration - -The `schedmd-slurm-gcp-v6-controller` module supports the reconfiguration of -partitions and slurm configuration in a running, active cluster. - -To reconfigure a running cluster: - -1. Edit the blueprint with the desired configuration changes -2. Call `gcluster create -w` to overwrite the deployment directory -3. Follow instructions in terminal to deploy - -The following are examples of updates that can be made to a running cluster: - -* Add or remove a partition to the cluster -* Resize an existing partition -* Attach new network storage to an existing partition - -> **NOTE**: Changing the VM `machine_type` of a partition may not work. -> It is better to create a new partition and delete the old one. - -## Custom Images - -For more information on creating valid custom images for the controller VM -instance or for custom instance templates, see our [vm-images.md] documentation -page. - -[vm-images.md]: ../../../../docs/vm-images.md#slurm-on-gcp-custom-images - -## GPU Support - -More information on GPU support in Slurm on GCP and other Cluster Toolkit modules -can be found at [docs/gpu-support.md](../../../../docs/gpu-support.md) - -## Reservation for Scheduled Maintenance - -A [maintenance event](https://cloud.google.com/compute/docs/instances/host-maintenance-overview#maintenanceevents) is when a compute engine stops a VM to perform a hardware or -software update which is determined by the host maintenance policy. This can -also affect the running jobs if the maintenance kicks in. Now, Customers can -protect jobs from getting terminated due to maintenance using the cluster -toolkit. You can enable creation of reservation for scheduled maintenance for -your compute nodeset and Slurm will reserve your node for maintenance during the -maintenance window. If you try to schedule any jobs which overlap with the -maintenance reservation, Slurm would not schedule any job. - -You can specify in your blueprint like - -```yaml - - id: compute_nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: [network] - settings: - enable_maintenance_reservation: true -``` - -To enable creation of reservation for maintenance. - -While running job on slurm cluster, you can specify total run time of the job -using [-t flag](https://slurm.schedmd.com/srun.html#OPT_time).This would only -run the job outside of the maintenance window. - -```shell -srun -n1 -pcompute -t 10:00 -``` - -Currently upcoming maintenance notification is supported in ALPHA version of -compute API. You can update the API version from your blueprint, - -```yaml - - id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - settings: - endpoint_versions: - compute: "alpha" -``` - -## Opportunistic GCP maintenance in Slurm - -Customers can also enable running GCP maintenance as Slurm job opportunistically -to perform early maintenance. If a node is detected for maintenance, Slurm will -create a job to perform maintenance and put it in the job queue. - -If [backfill](https://slurm.schedmd.com/sched_config.html#backfill) scheduler is -used, Slurm will backfill maintenance job if it can find any empty time window. - -Customer can also choose builtin scheduler type. In this case, Slurm would run -maintenance job in strictly priority order. If the maintenance job doesn't kick -in, then forced maintenance will take place at scheduled window. - -Customer can enable this feature at nodeset level by, - -```yaml - - id: debug_nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: [network] - settings: - enable_opportunistic_maintenance: true -``` - -## Placement Max Distance - -When using -[enable_placement](../../../../community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md#input_enable_placement) -with Slurm, Google Compute Engine will attempt to place VMs as physically close -together as possible. Capacity constraints at the time of VM creation may still -force VMs to be spread across multiple racks. Google provides the `max-distance` -flag which can used to control the maximum spreading allowed. Read more about -`max-distance` in the -[official docs](https://cloud.google.com/compute/docs/instances/use-compact-placement-policies -). - -You can use the `placement_max_distance` setting on the nodeset module to control the `max-distance` behavior. See the following example: - -```yaml - - id: nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: [ network ] - settings: - machine_type: c2-standard-4 - node_count_dynamic_max: 30 - enable_placement: true - placement_max_distance: 1 - -> [!NOTE] -> `schedmd-slurm-gcp-v6-nodeset.settings.enable_placement: true` must also be -> set for placement_max_distance to take effect. - -In the above case using a value of 1 will restrict VM to be placed on the same -rack. You can confirm that the `max-distance` was applied by calling the -following command while jobs are running: - -```shell -gcloud beta compute resource-policies list \ - --format='yaml(name,groupPlacementPolicy.maxDistance)' -``` - -> [!WARNING] -> If a zone lacks capacity, using a lower `max-distance` value (such as 1) is -> more likely to cause VMs creation to fail. - -## TreeWidth and Node Communication - -Slurm uses a fan out mechanism to communicate large groups of nodes. The shape -of this fan out tree is determined by the -[TreeWidth](https://slurm.schedmd.com/slurm.conf.html#OPT_TreeWidth) -configuration variable. - -In the cloud, this fan out mechanism can become unstable when nodes restart with -new IP addresses. You can enforce that all nodes communicate directly with the -controller by setting TreeWidth to a value >= largest partition. - -If the largest partition was 200 nodes, configure the blueprint as follows: - -```yaml - - id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - ... - settings: - cloud_parameters: - tree_width: 200 -``` - -The default has been set to 128. Values above this have not been fully tested -and may cause congestion on the controller. A more scalable solution is under -way. - -## ResumeRate and Node Resumption - -The `ResumeRate` parameter in `slurm.conf` controls the maximum number of nodes -that Slurm attempts to resume (power up) per minute. This is particularly -important in cloud environments where auto-scaling can lead to a large number of -nodes starting concurrently. - -When many nodes start simultaneously, they can place a heavy load on shared -resources, especially shared filesystems, as they all try to mount filesystems -and access configuration files at the same time. By limiting the `ResumeRate`, -you can stagger the node startup process, reducing the peak load on these shared -resources and improving overall cluster stability during scaling events. - -For example, to limit the node resumption rate to 100 nodes per minute, -configure the blueprint as follows: - -```yaml - - id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - ... - settings: - cloud_parameters: - resume_rate: 100 -``` - -Adjust this value based on the capabilities of your shared filesystem and the -expected scaling behavior of your cluster. - -## Support -The Cluster Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform -modules. For support with the underlying modules, see the instructions in the -[slurm-gcp README][slurm-gcp-readme]. - -[slurm-on-gcp]: https://github.com/GoogleCloudPlatform/slurm-gcp -[slurm-gcp-readme]: https://github.com/GoogleCloudPlatform/slurm-gcp#slurm-on-google-cloud-platform - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 4.84 | -| [google-beta](#requirement\_google-beta) | >= 6.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.84 | -| [google-beta](#provider\_google-beta) | >= 6.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [bucket](#module\_bucket) | terraform-google-modules/cloud-storage/google | ~> 6.1 | -| [daos\_network\_storage\_scripts](#module\_daos\_network\_storage\_scripts) | ../../../../modules/scripts/startup-script | n/a | -| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | -| [login](#module\_login) | ../../internal/slurm-gcp/login | n/a | -| [nodeset\_cleanup](#module\_nodeset\_cleanup) | ./modules/cleanup_compute | n/a | -| [nodeset\_cleanup\_tpu](#module\_nodeset\_cleanup\_tpu) | ./modules/cleanup_tpu | n/a | -| [slurm\_controller\_template](#module\_slurm\_controller\_template) | ../../internal/slurm-gcp/instance_template | n/a | -| [slurm\_files](#module\_slurm\_files) | ./modules/slurm_files | n/a | -| [slurm\_nodeset\_template](#module\_slurm\_nodeset\_template) | ../../internal/slurm-gcp/instance_template | n/a | -| [slurm\_nodeset\_tpu](#module\_slurm\_nodeset\_tpu) | ../../internal/slurm-gcp/nodeset_tpu | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_instance_from_template.controller](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance_from_template) | resource | -| [google_compute_disk.controller_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [google_secret_manager_secret.cloudsql](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret) | resource | -| [google_secret_manager_secret_iam_member.cloudsql_secret_accessor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_iam_member) | resource | -| [google_secret_manager_secret_version.cloudsql_version](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/secret_manager_secret_version) | resource | -| [google_storage_bucket_iam_member.legacy_readers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_member) | resource | -| [google_storage_bucket_iam_member.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_member) | resource | -| [google_storage_bucket_object.parition_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_compute_image.slurm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | -| [google_project.controller_project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    disk_name                  = string
    device_name                = string
    disk_type                  = string
    disk_size_gb               = number
    disk_labels                = map(string)
    auto_delete                = bool
    boot                       = bool
    disk_resource_manager_tags = map(string)
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | -| [bucket\_dir](#input\_bucket\_dir) | Bucket directory for cluster files to be put into. If not specified, then one will be chosen based on slurm\_cluster\_name. | `string` | `null` | no | -| [bucket\_name](#input\_bucket\_name) | Name of GCS bucket.
Ignored when 'create\_bucket' is true. | `string` | `null` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | -| [cgroup\_conf\_tpl](#input\_cgroup\_conf\_tpl) | Slurm cgroup.conf template file path. | `string` | `null` | no | -| [cloud\_parameters](#input\_cloud\_parameters) | cloud.conf options. Defaults inherited from [Slurm GCP repo](https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/terraform/slurm_cluster/modules/slurm_files/README_TF.md#input_cloud_parameters) | 
object({
    no_comma_params      = optional(bool, false)
    private_data         = optional(list(string))
    scheduler_parameters = optional(list(string))
    resume_rate          = optional(number)
    resume_timeout       = optional(number)
    suspend_rate         = optional(number)
    suspend_timeout      = optional(number)
    topology_plugin      = optional(string)
    topology_param       = optional(string)
    tree_width           = optional(number)
  })
| `{}` | no | -| [cloudsql](#input\_cloudsql) | Use this database instead of the one on the controller.
server\_ip : Address of the database server.
user : The user to access the database as.
password : The password, given the user, to access the given database. (sensitive)
db\_name : The database to access.
user\_managed\_replication : The list of location and (optional) kms\_key\_name for secret | 
object({
    server_ip = string
    user      = string
    password  = string # sensitive
    db_name   = string
    user_managed_replication = optional(list(object({
      location     = string
      kms_key_name = optional(string)
    })), [])
  })
| `null` | no | -| [compute\_startup\_script](#input\_compute\_startup\_script) | DEPRECATED: `compute_startup_script` has been deprecated.
Use `startup_script` of nodeset module instead. | `any` | `null` | no | -| [compute\_startup\_scripts\_timeout](#input\_compute\_startup\_scripts\_timeout) | The timeout (seconds) applied to each startup script in compute nodes. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [controller\_network\_attachment](#input\_controller\_network\_attachment) | SelfLink for NetworkAttachment to be attached to the controller, if any. | `string` | `null` | no | -| [controller\_project\_id](#input\_controller\_project\_id) | Optionally. Provision controller and config bucket in the different project | `string` | `null` | no | -| [controller\_startup\_script](#input\_controller\_startup\_script) | Startup script used by the controller VM. | `string` | `"# no-op"` | no | -| [controller\_startup\_scripts\_timeout](#input\_controller\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in controller\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [controller\_state\_disk](#input\_controller\_state\_disk) | A disk that will be attached to the controller instance template to save state of slurm. The disk is created and used by default.
To disable this feature, set this variable to null.

NOTE: This will not save the contents at /opt/apps and /home. To preserve those, they must be saved externally. | 
object({
    type = string
    size = number
  })
| 
{
  "size": 50,
  "type": "pd-ssd"
}
| no | -| [create\_bucket](#input\_create\_bucket) | Create GCS bucket instead of using an existing one. | `bool` | `true` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment. | `string` | n/a | yes | -| [disable\_controller\_public\_ips](#input\_disable\_controller\_public\_ips) | DEPRECATED: Use `enable_controller_public_ips` instead. | `bool` | `null` | no | -| [disable\_default\_mounts](#input\_disable\_default\_mounts) | DEPRECATED: Use `enable_default_mounts` instead. | `bool` | `null` | no | -| [disable\_smt](#input\_disable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | -| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | -| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `50` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-ssd"` | no | -| [enable\_bigquery\_load](#input\_enable\_bigquery\_load) | Enables loading of cluster job usage into big query.

NOTE: Requires Google Bigquery API. | `bool` | `false` | no | -| [enable\_chs\_gpu\_health\_check\_epilog](#input\_enable\_chs\_gpu\_health\_check\_epilog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as an epilog script after completing a job step from a new job allocation.
Compute nodes that fail GPU health check during epilog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | -| [enable\_chs\_gpu\_health\_check\_prolog](#input\_enable\_chs\_gpu\_health\_check\_prolog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as a prolog script whenever it is asked to run a job step from a new job allocation. Compute nodes that fail GPU health check during prolog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | -| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of compute nodes and resource policies (e.g.
placement groups) managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed compute nodes will be destroyed. | `bool` | `true` | no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_controller\_public\_ips](#input\_enable\_controller\_public\_ips) | If set to true. The controller will have a random public IP assigned to it. Ignored if access\_config is set. | `bool` | `false` | no | -| [enable\_debug\_logging](#input\_enable\_debug\_logging) | Enables debug logging mode. | `bool` | `false` | no | -| [enable\_default\_mounts](#input\_enable\_default\_mounts) | Enable default global network storage from the controller
- /home
- /apps | `bool` | `true` | no | -| [enable\_devel](#input\_enable\_devel) | DEPRECATED: `enable_devel` is always on. | `bool` | `null` | no | -| [enable\_external\_prolog\_epilog](#input\_enable\_external\_prolog\_epilog) | Automatically enable a script that will execute prolog and epilog scripts
shared by NFS from the controller to compute nodes. Find more details at:
https://github.com/GoogleCloudPlatform/slurm-gcp/blob/master/tools/prologs-epilogs/README.md | `bool` | `null` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_slurm\_auth](#input\_enable\_slurm\_auth) | Enables slurm authentication instead of munge. | `bool` | `false` | no | -| [enable\_slurm\_gcp\_plugins](#input\_enable\_slurm\_gcp\_plugins) | DEPRECATED: Slurm GCP plugins have been deprecated.
Instead of 'max\_hops' plugin please use the 'placement\_max\_distance' nodeset property.
Instead of 'enable\_vpmu' plugin please use 'advanced\_machine\_features.performance\_monitoring\_unit' nodeset property. | `any` | `null` | no | -| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| 
{
  "compute": "beta"
}
| no | -| [epilog\_scripts](#input\_epilog\_scripts) | List of scripts to be used for Epilog. Programs for the slurmd to execute
on every node when a user's job completes.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Epilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [extra\_logging\_flags](#input\_extra\_logging\_flags) | The only available flag is `trace_api` | `map(bool)` | `{}` | no | -| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | `""` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm controller VM instance.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-9-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | -| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | -| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for controller. | `string` | `null` | no | -| [labels](#input\_labels) | Labels, provided as a map. | `map(string)` | `{}` | no | -| [login\_network\_storage](#input\_login\_network\_storage) | An array of network attached storage mounts to be configured on all login nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string,
  }))
| `[]` | no | -| [login\_nodes](#input\_login\_nodes) | List of slurm login instance definitions. | 
list(object({
    group_name = string
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    additional_networks = optional(list(object({
      access_config = optional(list(object({
        nat_ip       = string
        network_tier = string
      })), [])
      alias_ip_range = optional(list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      })), [])
      ipv6_access_config = optional(list(object({
        network_tier = string
      })), [])
      network            = optional(string)
      network_ip         = optional(string, "")
      nic_type           = optional(string)
      queue_count        = optional(number)
      stack_type         = optional(string)
      subnetwork         = optional(string)
      subnetwork_project = optional(string)
    })), [])
    bandwidth_tier             = optional(string, "platform_default")
    can_ip_forward             = optional(bool, false)
    disk_auto_delete           = optional(bool, true)
    disk_labels                = optional(map(string), {})
    disk_resource_manager_tags = optional(map(string), {})
    disk_size_gb               = optional(number)
    disk_type                  = optional(string, "n1-standard-1")
    enable_confidential_vm     = optional(bool, false)
    enable_oslogin             = optional(bool, true)
    enable_shielded_vm         = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    metadata              = optional(map(string), {})
    min_cpu_platform      = optional(string)
    num_instances         = optional(number, 1)
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    static_ips           = optional(list(string), [])
    subnetwork           = string
    spot                 = optional(bool, false)
    tags                 = optional(list(string), [])
    zone                 = optional(string)
    termination_action   = optional(string)
  }))
| `[]` | no | -| [login\_startup\_script](#input\_login\_startup\_script) | Startup script used by the login VMs. | `string` | `"# no-op"` | no | -| [login\_startup\_scripts\_timeout](#input\_login\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in login\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"c2-standard-4"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on all instances. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = optional(map(string))
    mount_runner          = optional(map(string))
  }))
| `[]` | no | -| [nodeset](#input\_nodeset) | Define nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 1)
    node_conf              = optional(map(string), {})
    nodeset_name           = string
    additional_disks = optional(list(object({
      disk_name                  = optional(string)
      device_name                = optional(string)
      disk_size_gb               = optional(number)
      disk_type                  = optional(string)
      disk_labels                = optional(map(string), {})
      auto_delete                = optional(bool, true)
      boot                       = optional(bool, false)
      disk_resource_manager_tags = optional(map(string), {})
    })), [])
    bandwidth_tier                   = optional(string, "platform_default")
    can_ip_forward                   = optional(bool, false)
    disk_auto_delete                 = optional(bool, true)
    disk_labels                      = optional(map(string), {})
    disk_resource_manager_tags       = optional(map(string), {})
    disk_size_gb                     = optional(number)
    disk_type                        = optional(string)
    enable_confidential_vm           = optional(bool, false)
    enable_placement                 = optional(bool, false)
    placement_max_distance           = optional(number, null)
    enable_oslogin                   = optional(bool, true)
    enable_shielded_vm               = optional(bool, false)
    enable_maintenance_reservation   = optional(bool, false)
    enable_opportunistic_maintenance = optional(bool, false)
    gpu = optional(object({
      count = number
      type  = string
    }))
    dws_flex = object({
      enabled          = bool
      max_run_duration = number
      use_job_duration = bool
      use_bulk_insert  = bool
    })
    labels       = optional(map(string), {})
    machine_type = optional(string)
    advanced_machine_features = object({
      enable_nested_virtualization = optional(bool)
      threads_per_core             = optional(number)
      turbo_mode                   = optional(string)
      visible_core_count           = optional(number)
      performance_monitoring_unit  = optional(string)
      enable_uefi_networking       = optional(bool)
    })
    maintenance_interval     = optional(string)
    instance_properties_json = string
    metadata                 = optional(map(string), {})
    min_cpu_platform         = optional(string)
    network_tier             = optional(string, "STANDARD")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    on_host_maintenance   = optional(string)
    preemptible           = optional(bool, false)
    region                = optional(string)
    resource_manager_tags = optional(map(string), {})
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    shielded_instance_config = optional(object({
      enable_integrity_monitoring = optional(bool, true)
      enable_secure_boot          = optional(bool, true)
      enable_vtpm                 = optional(bool, true)
    }))
    source_image_family  = optional(string)
    source_image_project = optional(string)
    source_image         = optional(string)
    subnetwork_self_link = string
    additional_networks = optional(list(object({
      network            = string
      subnetwork         = string
      subnetwork_project = string
      network_ip         = string
      nic_type           = string
      stack_type         = string
      queue_count        = number
      access_config = list(object({
        nat_ip       = string
        network_tier = string
      }))
      ipv6_access_config = list(object({
        network_tier = string
      }))
      alias_ip_range = list(object({
        ip_cidr_range         = string
        subnetwork_range_name = string
      }))
    })))
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })))
    spot               = optional(bool, false)
    tags               = optional(list(string), [])
    termination_action = optional(string)
    reservation_name   = optional(string)
    future_reservation = string
    startup_script = optional(list(object({
      filename = string
    content = string })), [])

    zone_target_shape = string
    zone_policy_allow = set(string)
    zone_policy_deny  = set(string)
  }))
| `[]` | no | -| [nodeset\_dyn](#input\_nodeset\_dyn) | Defines dynamic nodesets, as a list. | 
list(object({
    nodeset_name    = string
    nodeset_feature = string
  }))
| `[]` | no | -| [nodeset\_tpu](#input\_nodeset\_tpu) | Define TPU nodesets, as a list. | 
list(object({
    node_count_static      = optional(number, 0)
    node_count_dynamic_max = optional(number, 5)
    nodeset_name           = string
    enable_public_ip       = optional(bool, false)
    node_type              = string
    accelerator_config = optional(object({
      topology = string
      version  = string
      }), {
      topology = ""
      version  = ""
    })
    tf_version   = string
    preemptible  = optional(bool, false)
    preserve_tpu = optional(bool, false)
    zone         = string
    data_disks   = optional(list(string), [])
    docker_image = optional(string, "")
    network_storage = optional(list(object({
      server_ip             = string
      remote_mount          = string
      local_mount           = string
      fs_type               = string
      mount_options         = string
      client_install_runner = optional(map(string))
      mount_runner          = optional(map(string))
    })), [])
    subnetwork = string
    service_account = optional(object({
      email  = optional(string)
      scopes = optional(list(string), ["https://www.googleapis.com/auth/cloud-platform"])
    }))
    project_id = string
    reserved   = optional(string, false)
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy. | `string` | `"MIGRATE"` | no | -| [partitions](#input\_partitions) | Cluster partitions as a list. See module slurm\_partition. | 
list(object({
    partition_name        = string
    partition_conf        = optional(map(string), {})
    partition_nodeset     = optional(list(string), [])
    partition_nodeset_dyn = optional(list(string), [])
    partition_nodeset_tpu = optional(list(string), [])
    enable_job_exclusive  = optional(bool, false)
  }))
| `[]` | no | -| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [prolog\_scripts](#input\_prolog\_scripts) | List of scripts to be used for Prolog. Programs for the slurmd to execute
whenever it is asked to run a job step from a new job allocation.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Prolog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [region](#input\_region) | The default region to place resources in. | `string` | n/a | yes | -| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the controller instance. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the controller instance. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used for resource naming and slurm accounting.
If not provided it will default to the first 8 characters of the deployment name (removing any invalid characters). | `string` | `null` | no | -| [slurm\_conf\_template](#input\_slurm\_conf\_template) | Slurm slurm.conf template. Content of the file in 'slurm\_conf\_tpl' is used if this is not set. | `string` | `null` | no | -| [slurm\_conf\_tpl](#input\_slurm\_conf\_tpl) | Slurm slurm.conf template file path. This path is used only if raw content is not provided in 'slurm\_conf\_template'. | `string` | `null` | no | -| [slurmdbd\_conf\_tpl](#input\_slurmdbd\_conf\_tpl) | Slurm slurmdbd.conf template file path. | `string` | `null` | no | -| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances. | `list(string)` | `[]` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | -| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | -| [task\_epilog\_scripts](#input\_task\_epilog\_scripts) | List of scripts to be used for TaskEpilog. Programs for the slurmd to execute
as the slurm job's owner after termination of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskEpilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [task\_prolog\_scripts](#input\_task\_prolog\_scripts) | List of scripts to be used for TaskProlog. Programs for the slurmd to execute
as the slurm job's owner prior to initiation of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskProlog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | `"googleapis.com"` | no | -| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be
spread across available zones in the region. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instructions](#output\_instructions) | Post deployment instructions. | -| [slurm\_bucket\_dir](#output\_slurm\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | -| [slurm\_bucket\_name](#output\_slurm\_bucket\_name) | GCS Bucket name of Slurm cluster file storage. | -| [slurm\_bucket\_path](#output\_slurm\_bucket\_path) | Bucket path used by cluster. | -| [slurm\_cluster\_name](#output\_slurm\_cluster\_name) | Slurm cluster name. | -| [slurm\_controller\_instance](#output\_slurm\_controller\_instance) | Compute instance of controller node | -| [slurm\_login\_instances](#output\_slurm\_login\_instances) | Compute instances of login nodes | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf deleted file mode 100644 index 4a887b99cf..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf +++ /dev/null @@ -1,213 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -module "gpu" { - source = "../../../../modules/internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - additional_disks = [ - for ad in var.additional_disks : { - disk_name = ad.disk_name - device_name = ad.device_name - disk_type = ad.disk_type - disk_size_gb = ad.disk_size_gb - disk_labels = merge(ad.disk_labels, local.labels) - auto_delete = ad.auto_delete - boot = ad.boot - disk_resource_manager_tags = ad.disk_resource_manager_tags - } - ] - - state_disk = var.controller_state_disk != null ? [{ - source = google_compute_disk.controller_disk[0].name - device_name = google_compute_disk.controller_disk[0].name - disk_labels = null - auto_delete = false - boot = false - }] : [] - - synth_def_sa_email = "${data.google_project.controller_project.number}-compute@developer.gserviceaccount.com" - - service_account = { - email = coalesce(var.service_account_email, local.synth_def_sa_email) - scopes = var.service_account_scopes - } - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - metadata = merge( - local.disable_automatic_updates_metadata, - var.metadata, - local.universe_domain - ) - - controller_project_id = coalesce(var.controller_project_id, var.project_id) -} - -data "google_project" "controller_project" { - project_id = local.controller_project_id -} - -resource "google_compute_disk" "controller_disk" { - count = var.controller_state_disk != null ? 1 : 0 - - project = local.controller_project_id - name = "${local.slurm_cluster_name}-controller-save" - type = var.controller_state_disk.type - size = var.controller_state_disk.size - zone = var.zone -} - -# INSTANCE TEMPLATE -module "slurm_controller_template" { - source = "../../internal/slurm-gcp/instance_template" - - project_id = local.controller_project_id - region = var.region - slurm_instance_role = "controller" - slurm_cluster_name = local.slurm_cluster_name - labels = local.labels - - disk_auto_delete = var.disk_auto_delete - disk_labels = merge(var.disk_labels, local.labels) - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - disk_resource_manager_tags = var.disk_resource_manager_tags - additional_disks = concat(local.additional_disks, local.state_disk) - - bandwidth_tier = var.bandwidth_tier - slurm_bucket_path = module.slurm_files.slurm_bucket_path - can_ip_forward = var.can_ip_forward - advanced_machine_features = var.advanced_machine_features - resource_manager_tags = var.resource_manager_tags - - enable_confidential_vm = var.enable_confidential_vm - enable_oslogin = var.enable_oslogin - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config - - gpu = one(module.gpu.guest_accelerator) - - machine_type = var.machine_type - metadata = local.metadata - min_cpu_platform = var.min_cpu_platform - - on_host_maintenance = var.on_host_maintenance - preemptible = var.preemptible - service_account = local.service_account - - source_image_family = local.source_image_family # requires source_image_logic.tf - source_image_project = local.source_image_project_normalized # requires source_image_logic.tf - source_image = local.source_image # requires source_image_logic.tf - - subnetwork = var.subnetwork_self_link - - tags = concat([local.slurm_cluster_name], var.tags) - # termination_action = TODO: add support for termination_action (?) -} - -# INSTANCE -resource "google_compute_instance_from_template" "controller" { - provider = google-beta - - name = "${local.slurm_cluster_name}-controller" - project = local.controller_project_id - zone = var.zone - source_instance_template = module.slurm_controller_template.self_link - # Due to https://github.com/hashicorp/terraform-provider-google/issues/21693 - # we have to explicitly override instance labels instead of inheriting them from template. - labels = module.slurm_controller_template.labels - - allow_stopping_for_update = true - - # Can't rely on template to specify nics due to usage of static_ip - network_interface { - dynamic "access_config" { - for_each = var.enable_controller_public_ips ? ["unit"] : [] - content { - nat_ip = null - network_tier = null - } - } - network_ip = length(var.static_ips) == 0 ? "" : var.static_ips[0] - subnetwork = var.subnetwork_self_link - } - - dynamic "network_interface" { - for_each = var.controller_network_attachment != null ? [1] : [] - content { - network_attachment = var.controller_network_attachment - } - } -} - -moved { - from = module.slurm_controller_instance.google_compute_instance_from_template.slurm_instance[0] - to = google_compute_instance_from_template.controller -} - -# SECRETS: CLOUDSQL -resource "google_secret_manager_secret" "cloudsql" { - count = var.cloudsql != null ? 1 : 0 - - secret_id = "${local.slurm_cluster_name}-slurm-secret-cloudsql" - project = var.project_id - - replication { - dynamic "auto" { - for_each = length(var.cloudsql.user_managed_replication) == 0 ? [1] : [] - content {} - } - dynamic "user_managed" { - for_each = length(var.cloudsql.user_managed_replication) == 0 ? [] : [1] - content { - dynamic "replicas" { - for_each = nonsensitive(var.cloudsql.user_managed_replication) - content { - location = replicas.value.location - dynamic "customer_managed_encryption" { - for_each = compact([replicas.value.kms_key_name]) - content { - kms_key_name = customer_managed_encryption.value - } - } - } - } - } - } - } - - labels = { - slurm_cluster_name = local.slurm_cluster_name - } -} - -resource "google_secret_manager_secret_version" "cloudsql_version" { - count = var.cloudsql != null ? 1 : 0 - - secret = google_secret_manager_secret.cloudsql[0].id - secret_data = jsonencode(var.cloudsql) -} - -resource "google_secret_manager_secret_iam_member" "cloudsql_secret_accessor" { - count = var.cloudsql != null ? 1 : 0 - - secret_id = google_secret_manager_secret.cloudsql[0].id - role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:${local.service_account.email}" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl deleted file mode 100644 index 219bdc5227..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurm.conf.tpl +++ /dev/null @@ -1,65 +0,0 @@ -# slurm.conf -# https://slurm.schedmd.com/high_throughput.html - -ProctrackType=proctrack/cgroup -SlurmctldPidFile=/var/run/slurm/slurmctld.pid -SlurmdPidFile=/var/run/slurm/slurmd.pid -TaskPlugin=task/affinity,task/cgroup -MaxArraySize=10001 -MaxJobCount=500000 -MaxNodeCount=65536 -MinJobAge=60 - -# -# -# SCHEDULING -SchedulerType=sched/backfill -SelectType=select/cons_tres -SelectTypeParameters=CR_Core_Memory - -# -# -# LOGGING AND ACCOUNTING -SlurmctldDebug=error -SlurmdDebug=error - -# -# -# TIMERS -MessageTimeout=60 - -################################################################################ -# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # -################################################################################ - -SlurmctldHost={control_host}({control_addr}) - -AuthType=auth/{auth_key} -AuthInfo=cred_expire=120 -AuthAltTypes=auth/jwt -CredType=cred/{auth_key} -MpiDefault={mpi_default} -ReturnToService=2 -SlurmctldPort={control_host_port} -SlurmdPort=6818 -SlurmdSpoolDir=/var/spool/slurmd -SlurmUser=slurm -StateSaveLocation={state_save} - -# -# -# LOGGING AND ACCOUNTING -AccountingStorageType=accounting_storage/slurmdbd -AccountingStorageHost={accounting_storage_host} -ClusterName={name} -SlurmctldLogFile={slurmlog}/slurmctld.log -SlurmdLogFile={slurmlog}/slurmd-%n.log - -# -# -# GENERATED CLOUD CONFIGURATIONS -include cloud.conf - -################################################################################ -# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # -################################################################################ diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl deleted file mode 100644 index 93ac47e341..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/htc-slurmdbd.conf.tpl +++ /dev/null @@ -1,34 +0,0 @@ -# slurmdbd.conf -# https://slurm.schedmd.com/slurmdbd.conf.html - -DebugLevel=info -PidFile=/var/run/slurm/slurmdbd.pid - -# https://slurm.schedmd.com/slurmdbd.conf.html#OPT_CommitDelay -CommitDelay=1 - -################################################################################ -# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # -################################################################################ - -AuthType=auth/{auth_key} -AuthAltTypes=auth/jwt -AuthAltParameters=jwt_key={state_save}/jwt_hs256.key - -DbdHost={control_host} - -LogFile={slurmlog}/slurmdbd.log - -SlurmUser=slurm - -StorageLoc={db_name} - -StorageType=accounting_storage/mysql -StorageHost={db_host} -StoragePort={db_port} -StorageUser={db_user} -StoragePass={db_pass} - -################################################################################ -# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # -################################################################################ diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl deleted file mode 100644 index d3f2615a68..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/etc/long-prolog-slurm.conf.tpl +++ /dev/null @@ -1,71 +0,0 @@ -# slurm.conf -# https://slurm.schedmd.com/slurm.conf.html -# https://slurm.schedmd.com/configurator.html - -ProctrackType=proctrack/cgroup -SlurmctldPidFile=/var/run/slurm/slurmctld.pid -SlurmdPidFile=/var/run/slurm/slurmd.pid -TaskPlugin=task/affinity,task/cgroup -MaxNodeCount=64000 - -# -# -# SCHEDULING -SchedulerType=sched/backfill -SelectType=select/cons_tres -SelectTypeParameters=CR_Core_Memory - -# -# -# LOGGING AND ACCOUNTING -AccountingStoreFlags=job_comment -JobAcctGatherFrequency=30 -JobAcctGatherType=jobacct_gather/cgroup -SlurmctldDebug=info -SlurmdDebug=info -DebugFlags=Power - -# -# -# TIMERS -MessageTimeout=600 -BatchStartTimeout=600 -PrologEpilogTimeout=600 -PrologFlags=Contain - -################################################################################ -# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # -################################################################################ - -SlurmctldHost={control_host}({control_addr}) - - -AuthType=auth/{auth_key} -AuthInfo=cred_expire=600 -AuthAltTypes=auth/jwt -CredType=cred/{auth_key} -MpiDefault={mpi_default} -ReturnToService=2 -SlurmctldPort={control_host_port} -SlurmdPort=6818 -SlurmdSpoolDir=/var/spool/slurmd -SlurmUser=slurm -StateSaveLocation={state_save} - -# -# -# LOGGING AND ACCOUNTING -AccountingStorageType=accounting_storage/slurmdbd -AccountingStorageHost={accounting_storage_host} -ClusterName={name} -SlurmctldLogFile={slurmlog}/slurmctld.log -SlurmdLogFile={slurmlog}/slurmd-%n.log - -# -# -# GENERATED CLOUD CONFIGURATIONS -include cloud.conf - -################################################################################ -# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # -################################################################################ diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf deleted file mode 100644 index 21e915a125..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -locals { - # TODO: deprecate `var.login_[ startup_script, startup_scripts_timeout, network_storage]` - # in favour of vars defined in user-facing login module - ghpc_startup_login = [{ - filename = "ghpc_startup.sh" - content = var.login_startup_script - }] - - login_startup_scripts = concat(local.common_scripts, local.ghpc_startup_login) -} - -module "login" { - source = "../../internal/slurm-gcp/login" - for_each = { for x in var.login_nodes : x.group_name => x } - - project_id = var.project_id - - slurm_cluster_name = local.slurm_cluster_name - slurm_bucket_path = module.slurm_files.slurm_bucket_path - slurm_bucket_name = module.slurm_files.bucket_name - slurm_bucket_dir = module.slurm_files.bucket_dir - - login_nodes = each.value - - startup_scripts = local.login_startup_scripts - startup_scripts_timeout = var.login_startup_scripts_timeout - - network_storage = var.login_network_storage - - universe_domain = var.universe_domain - - # trigger replacement of login nodes when the controller instance is replaced - # Needed for re-mounting volumes hosted on controller - replace_trigger = google_compute_instance_from_template.controller.self_link -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf deleted file mode 100644 index 7622bdffef..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/main.tf +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-controller", ghpc_role = "scheduler" }) -} - -locals { - # Since deployment name may be used to create a cluster name, we remove any invalid character from the beginning - # Also, slurm imposed a lot of restrictions to this name, so we format it to an acceptable string - tmp_cluster_name = substr(replace(lower(var.deployment_name), "/^[^a-z]*|[^a-z0-9]/", ""), 0, 10) - slurm_cluster_name = coalesce(var.slurm_cluster_name, local.tmp_cluster_name) - - universe_domain = { "universe_domain" = var.universe_domain } -} - -# See -# * slurm_files.tf -# * controller.tf -# * partition.tf -# * login.tf diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml deleted file mode 100644 index 7b4918b962..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - iam.googleapis.com - - storage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md deleted file mode 100644 index 002bf14145..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/README.md +++ /dev/null @@ -1,42 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [null](#provider\_null) | >= 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [null_resource.dependencies](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of compute nodes and resource policies (e.g.
placement groups) managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed compute nodes will be destroyed. | `bool` | n/a | yes | -| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | -| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | -| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name         = string
    subnetwork_self_link = string
    additional_networks = list(object({
      subnetwork = string
    }))
  })
| n/a | yes | -| [nodeset\_template](#input\_nodeset\_template) | Self link of the nodeset template | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf deleted file mode 100644 index bd8773cf84..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/main.tf +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - cleanup_dependencies_agg = flatten([ - var.nodeset.subnetwork_self_link, - var.nodeset.additional_networks[*].subnetwork, - var.nodeset_template]) -} - -# Can not use variadic list in `depends_on`, wrap it into a collection of `null_resource` -resource "null_resource" "dependencies" { - count = length(local.cleanup_dependencies_agg) -} - -resource "null_resource" "script" { - count = var.enable_cleanup_compute ? 1 : 0 - - triggers = { - project_id = var.project_id - cluster_name = var.slurm_cluster_name - nodeset_name = var.nodeset.nodeset_name - universe_domain = var.universe_domain - compute_endpoint_version = var.endpoint_versions.compute - gcloud_path_override = var.gcloud_path_override - } - - provisioner "local-exec" { - command = "/bin/bash ${path.module}/scripts/cleanup_compute.sh ${self.triggers.project_id} ${self.triggers.cluster_name} ${self.triggers.nodeset_name} ${self.triggers.universe_domain} ${self.triggers.compute_endpoint_version} ${self.triggers.gcloud_path_override}" - when = destroy - } - - # Ensure that clean up is done before attempt to delete the networks - depends_on = [null_resource.dependencies] -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh deleted file mode 100644 index 20d611969a..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/scripts/cleanup_compute.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/bash - -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -o pipefail - -project="$1" -cluster_name="$2" -nodeset_name="$3" -universe_domain="$4" -compute_endpoint_version="$5" -gcloud_dir="$6" - -if [[ $# -ne 5 ]] && [[ $# -ne 6 ]]; then - echo "Usage: $0 []" - exit 1 -fi - -if [[ -n "${gcloud_dir}" ]]; then - export PATH="$gcloud_dir:$PATH" -fi - -export CLOUDSDK_API_ENDPOINT_OVERRIDES_COMPUTE="https://www.${universe_domain}/compute/${compute_endpoint_version}/" -export CLOUDSDK_CORE_PROJECT="${project}" - -if ! type -P gcloud 1>/dev/null; then - echo "gcloud is not available and your compute resources are not being cleaned up" - echo "https://console.cloud.google.com/compute/instances?project=${project}" - exit 1 -fi - -tmpfile=$(mktemp) # have to use a temp file, since `< <(gcloud ...)` doesn't work nicely with `head` -trap 'rm -f "$tmpfile"' EXIT - -echo "Deleting managed instance groups" -mig_filter="name:${cluster_name}-${nodeset_name}-* AND status!=STOPPING" -gcloud compute instance-groups managed list --format="value(self_link)" --filter="${mig_filter}" >"$tmpfile" -while batch="$(head -n 2)" && [[ ${#batch} -gt 0 ]]; do - groups=$(echo "$batch" | paste -sd " " -) # concat into a single space-separated line - # The lack of quotes around ${groups} is intentional and causes each new space-separated "word" to - # be treated as independent arguments. See PR#2523 - # shellcheck disable=SC2086 - gcloud compute instance-groups managed delete --quiet ${groups} || echo "Failed to delete some instance groups" -done <"$tmpfile" -true >"$tmpfile" # Wipe contents of tmp file - -echo "Deleting compute nodes" -node_filter="name:${cluster_name}-${nodeset_name}-* labels.slurm_cluster_name=${cluster_name} AND labels.slurm_instance_role=compute" - -running_nodes_filter="${node_filter} AND status!=STOPPING" -# List all currently running instances and attempt to delete them -gcloud compute instances list --format="value(selfLink)" --filter="${running_nodes_filter}" >"$tmpfile" -# Do 500 instances at a time -while batch="$(head -n 500)" && [[ ${#batch} -gt 0 ]]; do - nodes=$(echo "$batch" | paste -sd " " -) # concat into a single space-separated line - # The lack of quotes around ${nodes} is intentional and causes each new space-separated "word" to - # be treated as independent arguments. See PR#2523 - # shellcheck disable=SC2086 - gcloud compute instances delete --quiet ${nodes} || echo "Failed to delete some instances" -done <"$tmpfile" - -# In case if controller tries to delete the nodes as well, -# wait until nodes in STOPPING state are deleted, before deleting the resource policies -stopping_nodes_filter="${node_filter} AND status=STOPPING" -while true; do - node=$(gcloud compute instances list --format="value(name)" --filter="${stopping_nodes_filter}" --limit=1) - if [[ -z "${node}" ]]; then - break - fi - echo "Waiting for instances to be deleted: ${node}" - sleep 5 -done - -echo "Deleting resource policies" -policies_filter="name:${cluster_name}-slurmgcp-managed-${nodeset_name}-*" -gcloud compute resource-policies list --format="value(selfLink)" --filter="${policies_filter}" | while read -r line; do - echo "Deleting resource policy: $line" - gcloud compute resource-policies delete --quiet "${line}" || { - echo "Failed to delete resource policy: $line" - } -done diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf deleted file mode 100644 index b6da69931c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_compute/variables.tf +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - type = string - description = "Project ID" -} - - -variable "slurm_cluster_name" { - type = string - description = "Name of the Slurm cluster" -} - -variable "enable_cleanup_compute" { - description = < [terraform](#requirement\_terraform) | >= 1.3 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [null](#provider\_null) | 3.2.3 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of TPU nodes managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed TPU nodes will be destroyed. | `bool` | n/a | yes | -| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | -| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | -| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name = string
    zone         = string
  })
| n/a | yes | -| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | - -## Outputs - -No outputs. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [null](#provider\_null) | >= 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [null_resource.script](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [enable\_cleanup\_compute](#input\_enable\_cleanup\_compute) | Enables automatic cleanup of TPU nodes managed by this module, when cluster is destroyed.

*WARNING*: Toggling this off will impact the running workload.
Deployed TPU nodes will be destroyed. | `bool` | n/a | yes | -| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| n/a | yes | -| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | n/a | yes | -| [nodeset](#input\_nodeset) | Nodeset to cleanup | 
object({
    nodeset_name = string
    zone         = string
  })
| n/a | yes | -| [project\_id](#input\_project\_id) | Project ID | `string` | n/a | yes | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Name of the Slurm cluster | `string` | n/a | yes | -| [universe\_domain](#input\_universe\_domain) | Domain address for alternate API universe | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf deleted file mode 100644 index ec86a03a24..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/main.tf +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resource "null_resource" "script" { - count = var.enable_cleanup_compute ? 1 : 0 - - triggers = { - project_id = var.project_id - cluster_name = var.slurm_cluster_name - nodeset_name = var.nodeset.nodeset_name - zone = var.nodeset.zone - universe_domain = var.universe_domain - compute_endpoint_version = var.endpoint_versions.compute - gcloud_path_override = var.gcloud_path_override - } - - provisioner "local-exec" { - command = "/bin/bash ${path.module}/scripts/cleanup_tpu.sh ${self.triggers.project_id} ${self.triggers.cluster_name} ${self.triggers.nodeset_name} ${self.triggers.zone} ${self.triggers.universe_domain} ${self.triggers.compute_endpoint_version} ${self.triggers.gcloud_path_override}" - when = destroy - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh deleted file mode 100644 index c724e342c3..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/scripts/cleanup_tpu.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -o pipefail - -project="$1" -cluster_name="$2" -nodeset_name="$3" -zone="$4" -universe_domain="$5" -compute_endpoint_version="$6" -gcloud_dir="$7" - -if [[ $# -ne 6 ]] && [[ $# -ne 7 ]]; then - echo "Usage: $0 []" - exit 1 -fi - -if [[ -n "${gcloud_dir}" ]]; then - export PATH="$gcloud_dir:$PATH" -fi - -export CLOUDSDK_API_ENDPOINT_OVERRIDES_COMPUTE="https://www.${universe_domain}/compute/${compute_endpoint_version}/" -export CLOUDSDK_CORE_PROJECT="${project}" - -if ! type -P gcloud 1>/dev/null; then - echo "gcloud is not available and your compute resources are not being cleaned up" - echo "https://console.cloud.google.com/compute/instances?project=${project}" - exit 1 -fi - -echo "Deleting TPU nodes" -node_filter="name~${cluster_name}-${nodeset_name}" -running_nodes_filter="${node_filter} AND state!=DELETING" - -# List all currently running nodes and attempt to delete them -gcloud compute tpus tpu-vm list --zone="${zone}" --format="value(name)" --filter="${running_nodes_filter}" | while read -r name; do - echo "Deleting TPU node: $name" - gcloud compute tpus tpu-vm delete --async --zone="${zone}" --quiet "${name}" || echo "Failed to delete $name" -done - -# Wait until nodes in DELETING state are deleted, before deleting the resource policies -deleting_nodes_filter="${node_filter} AND state=DELETING" -while true; do - node=$(gcloud compute tpus tpu-vm list --zone="${zone}" --format="value(name)" --filter="${deleting_nodes_filter}" --limit=1) - if [[ -z "${node}" ]]; then - break - fi - echo "Waiting for nodes to be deleted: ${node}" - sleep 5 -done diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf deleted file mode 100644 index 1ac6f64b75..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/cleanup_tpu/variables.tf +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright (C) Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - type = string - description = "Project ID" -} - -variable "slurm_cluster_name" { - type = string - description = "Name of the Slurm cluster" -} - -variable "enable_cleanup_compute" { - description = < -Copyright (C) SchedMD LLC. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | ~> 1.3 | -| [archive](#requirement\_archive) | ~> 2.0 | -| [google](#requirement\_google) | >= 3.53 | -| [local](#requirement\_local) | ~> 2.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [archive](#provider\_archive) | ~> 2.0 | -| [google](#provider\_google) | >= 3.53 | -| [local](#provider\_local) | ~> 2.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket_object.config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.controller_startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.devel](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.epilog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.nodeset_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.nodeset_dyn_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.nodeset_startup_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.nodeset_tpu_config](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.prolog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.task_epilog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [google_storage_bucket_object.task_prolog_scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [random_uuid.cluster_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/uuid) | resource | -| [archive_file.slurm_gcp_devel_zip](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | -| [google_storage_bucket.this](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | -| [local_file.chs_gpu_health_check](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | -| [local_file.external_epilog](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | -| [local_file.external_prolog](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | -| [local_file.setup_external](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [bucket\_dir](#input\_bucket\_dir) | Bucket directory for cluster files to be put into. | `string` | `null` | no | -| [bucket\_name](#input\_bucket\_name) | Name of GCS bucket to use. | `string` | n/a | yes | -| [cgroup\_conf\_tpl](#input\_cgroup\_conf\_tpl) | Slurm cgroup.conf template file path. | `string` | `null` | no | -| [cloud\_parameters](#input\_cloud\_parameters) | cloud.conf options. Default behavior defined in scripts/conf.py | 
object({
    no_comma_params      = optional(bool, false)
    private_data         = optional(list(string))
    scheduler_parameters = optional(list(string))
    resume_rate          = optional(number)
    resume_timeout       = optional(number)
    suspend_rate         = optional(number)
    suspend_timeout      = optional(number)
    topology_plugin      = optional(string)
    topology_param       = optional(string)
    tree_width           = optional(number)
  })
| `{}` | no | -| [cloudsql\_secret](#input\_cloudsql\_secret) | Secret URI to cloudsql secret. | `string` | `null` | no | -| [compute\_startup\_scripts\_timeout](#input\_compute\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in compute\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [controller\_network\_attachment](#input\_controller\_network\_attachment) | SelfLink for NetworkAttachment to be attached to the controller, if any. | `string` | `null` | no | -| [controller\_startup\_scripts](#input\_controller\_startup\_scripts) | List of scripts to be ran on controller VM startup. | 
list(object({
    filename = string
    content  = string
  }))
| `[]` | no | -| [controller\_startup\_scripts\_timeout](#input\_controller\_startup\_scripts\_timeout) | The timeout (seconds) applied to each script in controller\_startup\_scripts. If
any script exceeds this timeout, then the instance setup process is considered
failed and handled accordingly.

NOTE: When set to 0, the timeout is considered infinite and thus disabled. | `number` | `300` | no | -| [controller\_state\_disk](#input\_controller\_state\_disk) | A disk that will be attached to the controller instance template to save state of slurm. The disk is created and used by default.
To disable this feature, set this variable to null.

NOTE: This will not save the contents at /opt/apps and /home. To preserve those, they must be saved externally. | 
object({
    device_name = string
  })
| 
{
  "device_name": null
}
| no | -| [disable\_default\_mounts](#input\_disable\_default\_mounts) | Disable default global network storage from the controller
- /home
- /apps | `bool` | `false` | no | -| [enable\_bigquery\_load](#input\_enable\_bigquery\_load) | Enables loading of cluster job usage into big query.

NOTE: Requires Google Bigquery API. | `bool` | `false` | no | -| [enable\_chs\_gpu\_health\_check\_epilog](#input\_enable\_chs\_gpu\_health\_check\_epilog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as an epilog script after completing a job step from a new job allocation.
Compute nodes that fail GPU health check during epilog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | -| [enable\_chs\_gpu\_health\_check\_prolog](#input\_enable\_chs\_gpu\_health\_check\_prolog) | Enable a Cluster Health Sacnner(CHS) GPU health check that slurmd executes as a prolog script whenever it is asked to run a job step from a new job allocation. Compute nodes that fail GPU health check during prolog will be marked as drained. Find more details at:
https://github.com/GoogleCloudPlatform/cluster-toolkit/tree/main/docs/CHS-Slurm.md | `bool` | `false` | no | -| [enable\_debug\_logging](#input\_enable\_debug\_logging) | Enables debug logging mode. Not for production use. | `bool` | `false` | no | -| [enable\_external\_prolog\_epilog](#input\_enable\_external\_prolog\_epilog) | Automatically enable a script that will execute prolog and epilog scripts
shared by NFS from the controller to compute nodes. Find more details at:
https://github.com/GoogleCloudPlatform/slurm-gcp/blob/v5/tools/prologs-epilogs/README.md | `bool` | `false` | no | -| [enable\_hybrid](#input\_enable\_hybrid) | Enables use of hybrid controller mode. When true, controller\_hybrid\_config will
be used instead of controller\_instance\_config and will disable login instances. | `bool` | `false` | no | -| [enable\_slurm\_auth](#input\_enable\_slurm\_auth) | Enables slurm authentication instead of munge. | `bool` | `false` | no | -| [endpoint\_versions](#input\_endpoint\_versions) | Version of the API to use (The compute service is the only API currently supported) | 
object({
    compute = string
  })
| 
{
  "compute": null
}
| no | -| [epilog\_scripts](#input\_epilog\_scripts) | List of scripts to be used for Epilog. Programs for the slurmd to execute
on every node when a user's job completes.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Epilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [extra\_logging\_flags](#input\_extra\_logging\_flags) | The only available flag is `trace_api` | `map(bool)` | `{}` | no | -| [google\_app\_cred\_path](#input\_google\_app\_cred\_path) | Path to Google Application Credentials. | `string` | `null` | no | -| [install\_dir](#input\_install\_dir) | Directory where the hybrid configuration directory will be installed on the
on-premise controller (e.g. /etc/slurm/hybrid). This updates the prefix path
for the resume and suspend scripts in the generated `cloud.conf` file.

This variable should be used when the TerraformHost and the SlurmctldHost
are different.

This will default to var.output\_dir if null. | `string` | `null` | no | -| [munge\_mount](#input\_munge\_mount) | Remote munge mount for compute and login nodes to acquire the munge.key.
By default, the munge mount server will be assumed to be the
`var.slurm_control_host` (or `var.slurm_control_addr` if non-null) when
`server_ip=null`. | 
object({
    server_ip     = string
    remote_mount  = string
    fs_type       = string
    mount_options = string
  })
| 
{
  "fs_type": "nfs",
  "mount_options": "",
  "remote_mount": "/etc/munge/",
  "server_ip": null
}
| no | -| [network\_storage](#input\_network\_storage) | Storage to mounted on all instances.
- server\_ip : Address of the storage server.
- remote\_mount : The location in the remote instance filesystem to mount from.
- local\_mount : The location on the instance filesystem to mount to.
- fs\_type : Filesystem type (e.g. "nfs").
- mount\_options : Options to mount with. | 
list(object({
    server_ip     = string
    remote_mount  = string
    local_mount   = string
    fs_type       = string
    mount_options = string
  }))
| `[]` | no | -| [nodeset](#input\_nodeset) | Cluster nodenets, as a list. | `list(any)` | `[]` | no | -| [nodeset\_dyn](#input\_nodeset\_dyn) | Cluster nodenets (dynamic), as a list. | `list(any)` | `[]` | no | -| [nodeset\_startup\_scripts](#input\_nodeset\_startup\_scripts) | List of scripts to be ran on compute VM startup in the specific nodeset. | 
map(list(object({
    filename = string
    content  = string
  })))
| `{}` | no | -| [nodeset\_tpu](#input\_nodeset\_tpu) | Cluster nodenets (TPU), as a list. | `list(any)` | `[]` | no | -| [output\_dir](#input\_output\_dir) | Directory where this module will write its files to. These files include:
cloud.conf; cloud\_gres.conf; config.yaml; resume.py; suspend.py; and util.py. | `string` | `null` | no | -| [project\_id](#input\_project\_id) | The GCP project ID. | `string` | n/a | yes | -| [prolog\_scripts](#input\_prolog\_scripts) | List of scripts to be used for Prolog. Programs for the slurmd to execute
whenever it is asked to run a job step from a new job allocation.
See https://slurm.schedmd.com/slurm.conf.html#OPT_Prolog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [slurm\_bin\_dir](#input\_slurm\_bin\_dir) | Path to directory of Slurm binary commands (e.g. scontrol, sinfo). If 'null',
then it will be assumed that binaries are in $PATH. | `string` | `null` | no | -| [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | The cluster name, used for resource naming and slurm accounting. | `string` | n/a | yes | -| [slurm\_conf\_template](#input\_slurm\_conf\_template) | Slurm slurm.conf template. Content of the file in 'slurm\_conf\_tpl' is used if this is not set. | `string` | `null` | no | -| [slurm\_conf\_tpl](#input\_slurm\_conf\_tpl) | Slurm slurm.conf template file path. This path is used only if raw content is not provided in 'slurm\_conf\_template'. | `string` | `null` | no | -| [slurm\_control\_addr](#input\_slurm\_control\_addr) | The IP address or a name by which the address can be identified.

This value is passed to slurm.conf such that:
SlurmctldHost={var.slurm\_control\_host}\({var.slurm\_control\_addr}\)

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldHost | `string` | `null` | no | -| [slurm\_control\_host](#input\_slurm\_control\_host) | The short, or long, hostname of the machine where Slurm control daemon is
executed (i.e. the name returned by the command "hostname -s").

This value is passed to slurm.conf such that:
SlurmctldHost={var.slurm\_control\_host}\({var.slurm\_control\_addr}\)

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldHost | `string` | `null` | no | -| [slurm\_control\_host\_port](#input\_slurm\_control\_host\_port) | The port number that the Slurm controller, slurmctld, listens to for work.

See https://slurm.schedmd.com/slurm.conf.html#OPT_SlurmctldPort | `string` | `"6818"` | no | -| [slurm\_key\_mount](#input\_slurm\_key\_mount) | Remote mount for compute and login nodes to acquire the slurm.key. | 
object({
    server_ip     = string
    remote_mount  = string
    fs_type       = string
    mount_options = string
  })
| `null` | no | -| [slurm\_log\_dir](#input\_slurm\_log\_dir) | Directory where Slurm logs to. | `string` | `"/var/log/slurm"` | no | -| [slurmdbd\_conf\_tpl](#input\_slurmdbd\_conf\_tpl) | Slurm slurmdbd.conf template file path. | `string` | `null` | no | -| [task\_epilog\_scripts](#input\_task\_epilog\_scripts) | List of scripts to be used for TaskEpilog. Programs for the slurmd to execute
as the slurm job's owner after termination of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskEpilog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | -| [task\_prolog\_scripts](#input\_task\_prolog\_scripts) | List of scripts to be used for TaskProlog. Programs for the slurmd to execute
as the slurm job's owner prior to initiation of each task.
See https://slurm.schedmd.com/slurm.conf.html#OPT_TaskProlog. | 
list(object({
    filename = string
    content  = optional(string)
    source   = optional(string)
  }))
| `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [bucket\_dir](#output\_bucket\_dir) | Path directory within `bucket_name` for Slurm cluster file storage. | -| [bucket\_name](#output\_bucket\_name) | GCS Bucket name of Slurm cluster file storage. | -| [config](#output\_config) | Cluster configuration. | -| [slurm\_bucket\_path](#output\_slurm\_bucket\_path) | GCS Bucket URI of Slurm cluster file storage. | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl deleted file mode 100644 index ffeb167cfc..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/cgroup.conf.tpl +++ /dev/null @@ -1,7 +0,0 @@ -# cgroup.conf -# https://slurm.schedmd.com/cgroup.conf.html - -ConstrainCores=yes -ConstrainRamSpace=yes -ConstrainSwapSpace=no -ConstrainDevices=yes diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl deleted file mode 100644 index 4951289842..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurm.conf.tpl +++ /dev/null @@ -1,67 +0,0 @@ -# slurm.conf -# https://slurm.schedmd.com/slurm.conf.html -# https://slurm.schedmd.com/configurator.html - -ProctrackType=proctrack/cgroup -SlurmctldPidFile=/var/run/slurm/slurmctld.pid -SlurmdPidFile=/var/run/slurm/slurmd.pid -TaskPlugin=task/affinity,task/cgroup -MaxNodeCount=64000 - -# -# -# SCHEDULING -SchedulerType=sched/backfill -SelectType=select/cons_tres -SelectTypeParameters=CR_Core_Memory - -# -# -# LOGGING AND ACCOUNTING -AccountingStoreFlags=job_comment -JobAcctGatherFrequency=30 -JobAcctGatherType=jobacct_gather/cgroup -SlurmctldDebug=info -SlurmdDebug=info -DebugFlags=Power - -# -# -# TIMERS -MessageTimeout=60 - -################################################################################ -# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # -################################################################################ - -SlurmctldHost={control_host}({control_addr}) - -AuthType=auth/{auth_key} -AuthInfo=cred_expire=120 -AuthAltTypes=auth/jwt -CredType=cred/{auth_key} -MpiDefault={mpi_default} -ReturnToService=2 -SlurmctldPort={control_host_port} -SlurmdPort=6818 -SlurmdSpoolDir=/var/spool/slurmd -SlurmUser=slurm -StateSaveLocation={state_save} - -# -# -# LOGGING AND ACCOUNTING -AccountingStorageType=accounting_storage/slurmdbd -AccountingStorageHost={accounting_storage_host} -ClusterName={name} -SlurmctldLogFile={slurmlog}/slurmctld.log -SlurmdLogFile={slurmlog}/slurmd-%n.log - -# -# -# GENERATED CLOUD CONFIGURATIONS -include cloud.conf - -################################################################################ -# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # -################################################################################ diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl deleted file mode 100644 index 8c90a9dfbe..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/etc/slurmdbd.conf.tpl +++ /dev/null @@ -1,31 +0,0 @@ -# slurmdbd.conf -# https://slurm.schedmd.com/slurmdbd.conf.html - -DebugLevel=info -PidFile=/var/run/slurm/slurmdbd.pid - -################################################################################ -# vvvvv WARNING: DO NOT MODIFY SECTION BELOW vvvvv # -################################################################################ - -AuthType=auth/{auth_key} -AuthAltTypes=auth/jwt -AuthAltParameters=jwt_key={state_save}/jwt_hs256.key - -DbdHost={control_host} - -LogFile={slurmlog}/slurmdbd.log - -SlurmUser=slurm - -StorageLoc={db_name} - -StorageType=accounting_storage/mysql -StorageHost={db_host} -StoragePort={db_port} -StorageUser={db_user} -StoragePass={db_pass} - -################################################################################ -# ^^^^^ WARNING: DO NOT MODIFY SECTION ABOVE ^^^^^ # -################################################################################ diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh deleted file mode 100644 index db514fc9e5..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_epilog.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [[ -x /opt/apps/adm/slurm/slurm_epilog ]]; then - exec /opt/apps/adm/slurm/slurm_epilog -fi diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh deleted file mode 100644 index 37a91bb1ea..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/external_prolog.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [[ -x /opt/apps/adm/slurm/slurm_prolog ]]; then - exec /opt/apps/adm/slurm/slurm_prolog -fi diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh deleted file mode 100644 index 21454bd52c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/files/setup_external.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -SLURM_EXTERNAL_ROOT="/opt/apps/adm/slurm" -SLURM_MUX_FILE="slurm_mux" - -mkdir -p "${SLURM_EXTERNAL_ROOT}" -mkdir -p "${SLURM_EXTERNAL_ROOT}/logs" -mkdir -p "${SLURM_EXTERNAL_ROOT}/etc" - -# create common prolog / epilog "multiplex" script -if [ ! -f "${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" ]; then - # indentation matters in EOT below; do not blindly edit! - cat <<'EOT' >"${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -CMD="${0##*/}" -# Locate script -BASE=$(readlink -f $0) -BASE=${BASE%/*} - -export CLUSTER_ADM_BASE=${BASE} - -# Source config file if it exists for extra DEBUG settings -# used below -SLURM_MUX_CONF=${CLUSTER_ADM_BASE}/etc/slurm_mux.conf -if [[ -r ${SLURM_MUX_CONF} ]]; then - source ${SLURM_MUX_CONF} -fi - -# Setup logging if configured and directory exists -LOGFILE="/dev/null" -if [[ -d ${DEBUG_SLURM_MUX_LOG_DIR} && ${DEBUG_SLURM_MUX_ENABLE_LOG} == "yes" ]]; then - LOGFILE="${DEBUG_SLURM_MUX_LOG_DIR}/${CMD}-${SLURM_SCRIPT_CONTEXT}-job-${SLURMD_NODENAME}.log" - exec >>${LOGFILE} 2>&1 -fi - -# Global scriptlets -for SCRIPTLET in ${BASE}/${SLURM_SCRIPT_CONTEXT}.d/*.${SLURM_SCRIPT_CONTEXT}; do - if [[ -x ${SCRIPTLET} ]]; then - echo "Running ${SCRIPTLET}" - ${SCRIPTLET} $@ >>${LOGFILE} 2>&1 - fi -done - -# Per partition scriptlets -for SCRIPTLET in ${BASE}/partition-${SLURM_JOB_PARTITION}-${SLURM_SCRIPT_CONTEXT}.d/*.${SLURM_SCRIPT_CONTEXT}; do - if [[ -x ${SCRIPTLET} ]]; then - echo "Running ${SCRIPTLET}" - ${SCRIPTLET} $@ >>${LOGFILE} 2>&1 - fi -done -EOT -fi - -# ensure proper permissions on slurm_mux script -chmod 0755 "${SLURM_EXTERNAL_ROOT}/${SLURM_MUX_FILE}" - -# create default slurm_mux configuration file -if [ ! -f "${SLURM_EXTERNAL_ROOT}/etc/slurm_mux.conf" ]; then - cat <<'EOT' >"${SLURM_EXTERNAL_ROOT}/etc/slurm_mux.conf" -# these settings are intended for temporary debugging purposes only; leaving -# them enabled will write files for each job to a shared NFS directory without -# any automated cleanup -DEBUG_SLURM_MUX_LOG_DIR=/opt/apps/adm/slurm/logs -DEBUG_SLURM_MUX_ENABLE_LOG=no -EOT -fi - -# create epilog symbolic link -if [ ! -L "${SLURM_EXTERNAL_ROOT}/slurm_epilog" ]; then - cd ${SLURM_EXTERNAL_ROOT} - # delete existing file if necessary - rm -f slurm_epilog - ln -s ${SLURM_MUX_FILE} slurm_epilog - cd - >/dev/null -fi - -# create prolog symbolic link -if [ ! -L "${SLURM_EXTERNAL_ROOT}/slurm_prolog" ]; then - cd ${SLURM_EXTERNAL_ROOT} - # delete existing file if necessary - rm -f slurm_prolog - ln -s ${SLURM_MUX_FILE} slurm_prolog - cd - >/dev/null -fi diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf deleted file mode 100644 index a4855cb67c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/main.tf +++ /dev/null @@ -1,331 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - scripts_dir = abspath("${path.module}/scripts") - - bucket_dir = coalesce(var.bucket_dir, format("%s-files", var.slurm_cluster_name)) -} - -######## -# DATA # -######## - -data "google_storage_bucket" "this" { - name = var.bucket_name -} - -########## -# RANDOM # -########## - -resource "random_uuid" "cluster_id" { -} - -################## -# CLUSTER CONFIG # -################## - -locals { - config = { - enable_bigquery_load = var.enable_bigquery_load - cloudsql_secret = var.cloudsql_secret - cluster_id = random_uuid.cluster_id.result - project = var.project_id - slurm_cluster_name = var.slurm_cluster_name - enable_slurm_auth = var.enable_slurm_auth - bucket_path = local.bucket_path - enable_debug_logging = var.enable_debug_logging - extra_logging_flags = var.extra_logging_flags - controller_state_disk = var.controller_state_disk - - # storage - disable_default_mounts = var.disable_default_mounts - network_storage = var.network_storage - - # timeouts - controller_startup_scripts_timeout = var.controller_startup_scripts_timeout - compute_startup_scripts_timeout = var.compute_startup_scripts_timeout - - munge_mount = local.munge_mount - slurm_key_mount = var.slurm_key_mount - - # slurm conf - prolog_scripts = [for k, v in google_storage_bucket_object.prolog_scripts : k] - epilog_scripts = [for k, v in google_storage_bucket_object.epilog_scripts : k] - task_prolog_scripts = [for k, v in google_storage_bucket_object.task_prolog_scripts : k] - task_epilog_scripts = [for k, v in google_storage_bucket_object.task_epilog_scripts : k] - cloud_parameters = var.cloud_parameters - - # hybrid - hybrid = var.enable_hybrid - google_app_cred_path = var.enable_hybrid ? local.google_app_cred_path : null - output_dir = var.enable_hybrid ? local.output_dir : null - install_dir = var.enable_hybrid ? local.install_dir : null - slurm_control_host = var.enable_hybrid ? var.slurm_control_host : null - slurm_control_host_port = var.enable_hybrid ? local.slurm_control_host_port : null - slurm_control_addr = var.enable_hybrid ? var.slurm_control_addr : null - slurm_bin_dir = var.enable_hybrid ? local.slurm_bin_dir : null - slurm_log_dir = var.enable_hybrid ? local.slurm_log_dir : null - controller_network_attachment = var.controller_network_attachment - - - # config files templates - slurmdbd_conf_tpl = file(coalesce(var.slurmdbd_conf_tpl, "${local.etc_dir}/slurmdbd.conf.tpl")) - slurm_conf_tpl = var.slurm_conf_template != null ? var.slurm_conf_template : file(coalesce(var.slurm_conf_tpl, "${local.etc_dir}/slurm.conf.tpl")) - cgroup_conf_tpl = file(coalesce(var.cgroup_conf_tpl, "${local.etc_dir}/cgroup.conf.tpl")) - - # Providers - endpoint_versions = var.endpoint_versions - } - - x_nodeset = toset(var.nodeset[*].nodeset_name) - x_nodeset_dyn = toset(var.nodeset_dyn[*].nodeset_name) - x_nodeset_tpu = toset(var.nodeset_tpu[*].nodeset.nodeset_name) - x_nodeset_overlap = setintersection([], local.x_nodeset, local.x_nodeset_dyn, local.x_nodeset_tpu) - - etc_dir = abspath("${path.module}/etc") - - bucket_path = format("%s/%s", data.google_storage_bucket.this.url, local.bucket_dir) - - slurm_control_host_port = coalesce(var.slurm_control_host_port, "6818") - - google_app_cred_path = var.google_app_cred_path != null ? abspath(var.google_app_cred_path) : null - slurm_bin_dir = var.slurm_bin_dir != null ? abspath(var.slurm_bin_dir) : null - slurm_log_dir = var.slurm_log_dir != null ? abspath(var.slurm_log_dir) : null - - munge_mount = var.enable_hybrid ? { - server_ip = lookup(var.munge_mount, "server_ip", coalesce(var.slurm_control_addr, var.slurm_control_host)) - remote_mount = lookup(var.munge_mount, "remote_mount", "/etc/munge/") - fs_type = lookup(var.munge_mount, "fs_type", "nfs") - mount_options = lookup(var.munge_mount, "mount_options", "") - } : null - - output_dir = can(coalesce(var.output_dir)) ? abspath(var.output_dir) : abspath(".") - install_dir = can(coalesce(var.install_dir)) ? abspath(var.install_dir) : local.output_dir -} - -resource "google_storage_bucket_object" "config" { - bucket = data.google_storage_bucket.this.name - name = "${local.bucket_dir}/config.yaml" - content = yamlencode(local.config) - - # Take dependency on all other "config artifacts" so creation of `config.yaml` - # can be used as a signal for setup.py that "everything is ready". - # Some of following files, particularly mount scripts for new NFSes, can take a while to be created. - depends_on = [ - google_storage_bucket_object.controller_startup_scripts, - google_storage_bucket_object.nodeset_startup_scripts, - google_storage_bucket_object.prolog_scripts, - google_storage_bucket_object.epilog_scripts, - google_storage_bucket_object.task_prolog_scripts, - google_storage_bucket_object.task_epilog_scripts - ] -} - -resource "google_storage_bucket_object" "nodeset_config" { - for_each = { for ns in var.nodeset : ns.nodeset_name => merge(ns, { - instance_properties = jsondecode(ns.instance_properties_json) - }) } - - bucket = data.google_storage_bucket.this.name - name = "${local.bucket_dir}/nodeset_configs/${each.key}.yaml" - content = yamlencode(each.value) -} - -resource "google_storage_bucket_object" "nodeset_dyn_config" { - for_each = { for ns in var.nodeset_dyn : ns.nodeset_name => ns } - - bucket = data.google_storage_bucket.this.name - name = "${local.bucket_dir}/nodeset_dyn_configs/${each.key}.yaml" - content = yamlencode(each.value) -} - -resource "google_storage_bucket_object" "nodeset_tpu_config" { - for_each = { for n in var.nodeset_tpu[*].nodeset : n.nodeset_name => n } - - bucket = data.google_storage_bucket.this.name - name = "${local.bucket_dir}/nodeset_tpu_configs/${each.key}.yaml" - content = yamlencode(each.value) -} - -######### -# DEVEL # -######### - -locals { - build_dir = abspath("${path.module}/build") - - slurm_gcp_devel_zip = "slurm-gcp-devel.zip" - slurm_gcp_devel_zip_bucket = format("%s/%s", local.bucket_dir, local.slurm_gcp_devel_zip) -} - -data "archive_file" "slurm_gcp_devel_zip" { - output_path = "${local.build_dir}/${local.slurm_gcp_devel_zip}" - type = "zip" - source_dir = local.scripts_dir - - excludes = flatten([ - fileset(local.scripts_dir, "tests/**"), - # TODO: consider removing (including nested) __pycache__ and all .* files - # Though it only affects developers - ]) - -} - -resource "google_storage_bucket_object" "devel" { - bucket = var.bucket_name - name = local.slurm_gcp_devel_zip_bucket - source = data.archive_file.slurm_gcp_devel_zip.output_path -} - - -########### -# SCRIPTS # -########### - -resource "google_storage_bucket_object" "controller_startup_scripts" { - for_each = { - for x in local.controller_startup_scripts - : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x - } - - bucket = var.bucket_name - name = format("%s/slurm-controller-script-%s", local.bucket_dir, each.key) - content = each.value.content -} - -resource "google_storage_bucket_object" "nodeset_startup_scripts" { - for_each = { for x in flatten([ - for nodeset, scripts in var.nodeset_startup_scripts - : [for s in scripts - : { - content = s.content, - name = format("slurm-nodeset-%s-script-%s", nodeset, replace(basename(s.filename), "/[^a-zA-Z0-9-_]/", "_")) } - ]]) : x.name => x.content } - - bucket = var.bucket_name - name = format("%s/%s", local.bucket_dir, each.key) - content = each.value -} - -resource "google_storage_bucket_object" "prolog_scripts" { - for_each = { - for x in local.prolog_scripts - : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x - } - - bucket = var.bucket_name - name = format("%s/slurm-prolog-script-%s", local.bucket_dir, each.key) - content = each.value.content - source = each.value.source -} - -resource "google_storage_bucket_object" "epilog_scripts" { - for_each = { - for x in local.epilog_scripts - : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x - } - - bucket = var.bucket_name - name = format("%s/slurm-epilog-script-%s", local.bucket_dir, each.key) - content = each.value.content - source = each.value.source -} - -resource "google_storage_bucket_object" "task_prolog_scripts" { - for_each = { - for x in local.task_prolog_scripts - : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x - } - - bucket = var.bucket_name - name = format("%s/slurm-task_prolog-script-%s", local.bucket_dir, each.key) - content = each.value.content - source = each.value.source -} - -resource "google_storage_bucket_object" "task_epilog_scripts" { - for_each = { - for x in local.task_epilog_scripts - : replace(basename(x.filename), "/[^a-zA-Z0-9-_]/", "_") => x - } - - bucket = var.bucket_name - name = format("%s/slurm-task_epilog-script-%s", local.bucket_dir, each.key) - content = each.value.content - source = each.value.source -} - -############################ -# DATA: CHS GPU HEALTH CHECK -############################ - -data "local_file" "chs_gpu_health_check" { - filename = "${path.module}/scripts/tools/gpu-test" -} - -################################ -# DATA: EXTERNAL PROLOG/EPILOG # -################################ - -data "local_file" "external_epilog" { - filename = "${path.module}/files/external_epilog.sh" -} - -data "local_file" "external_prolog" { - filename = "${path.module}/files/external_prolog.sh" -} - -data "local_file" "setup_external" { - filename = "${path.module}/files/setup_external.sh" -} - -locals { - external_epilog = [{ - filename = "z_external_epilog.sh" - content = data.local_file.external_epilog.content - source = null - }] - external_prolog = [{ - filename = "z_external_prolog.sh" - content = data.local_file.external_prolog.content - source = null - }] - setup_external = [{ - filename = "z_setup_external.sh" - content = data.local_file.setup_external.content - }] - chs_gpu_health_check = [{ - filename = "a_chs_gpu_health_check.sh" - content = data.local_file.chs_gpu_health_check.content - source = null - }] - - chs_prolog = var.enable_chs_gpu_health_check_prolog ? local.chs_gpu_health_check : [] - ext_prolog = var.enable_external_prolog_epilog ? local.external_prolog : [] - prolog_scripts = concat(local.chs_prolog, local.ext_prolog, var.prolog_scripts) - task_prolog_scripts = var.task_prolog_scripts - - chs_epilog = var.enable_chs_gpu_health_check_epilog ? local.chs_gpu_health_check : [] - ext_epilog = var.enable_external_prolog_epilog ? local.external_epilog : [] - epilog_scripts = concat(local.chs_epilog, local.ext_epilog, var.epilog_scripts) - task_epilog_scripts = var.task_epilog_scripts - - controller_startup_scripts = var.enable_external_prolog_epilog ? concat(local.setup_external, var.controller_startup_scripts) : var.controller_startup_scripts - - -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf deleted file mode 100644 index 111c997d62..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/outputs.tf +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "slurm_bucket_path" { - description = "GCS Bucket URI of Slurm cluster file storage." - value = local.bucket_path -} - -output "bucket_name" { - description = "GCS Bucket name of Slurm cluster file storage." - value = data.google_storage_bucket.this.name -} - -output "bucket_dir" { - description = "Path directory within `bucket_name` for Slurm cluster file storage." - value = local.bucket_dir -} - -output "config" { - description = "Cluster configuration." - value = local.config - - precondition { - condition = var.enable_hybrid ? can(coalesce(var.slurm_control_host)) : true - error_message = "Input slurm_control_host is required." - } - - precondition { - condition = length(local.x_nodeset_overlap) == 0 - error_message = "All nodeset names must be unique among all nodeset types." - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py deleted file mode 100644 index e063093800..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/conf.py +++ /dev/null @@ -1,641 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Optional, Iterable, Dict, Set, Tuple -from itertools import chain -from collections import defaultdict -import json -from pathlib import Path -import util -from util import dirs, slurmdirs -import tpu -from addict import Dict as NSDict # type: ignore - -FILE_PREAMBLE = """ -# Warning: -# This file is managed by a script. Manual modifications will be overwritten. -""" - - - -def dict_to_conf(conf, delim=" ") -> str: - """convert dict to delimited slurm-style key-value pairs""" - - def filter_conf(pair): - k, v = pair - if isinstance(v, list): - v = ",".join(str(el) for el in v if el is not None) - return k, (v if bool(v) or v == 0 else None) - - return delim.join( - f"{k}={v}" for k, v in map(filter_conf, conf.items()) if v is not None - ) - - -TOPOLOGY_PLUGIN_TREE = "topology/tree" - -def topology_plugin(lkp: util.Lookup) -> str: - """ - Returns configured topology plugin, defaults to `topology/tree`. - """ - cp, key = lkp.cfg.cloud_parameters, "topology_plugin" - if key not in cp or cp[key] is None: - return TOPOLOGY_PLUGIN_TREE - return cp[key] - -def conflines(lkp: util.Lookup) -> str: - params = lkp.cfg.cloud_parameters - def get(key, default): - """ - Returns the value of the key in params if it exists and is not None, - otherwise returns supplied default. - We can't rely on the `dict.get` method because the value could be `None` as - well as empty NSDict, depending on type of the `cfg.cloud_parameters`. - TODO: Simplify once NSDict is removed from the codebase. - """ - if key not in params or params[key] is None: - return default - return params[key] - - no_comma_params = get("no_comma_params", False) - - any_gpus = any( - lkp.template_info(nodeset.instance_template).gpu - for nodeset in lkp.cfg.nodeset.values() - ) - - any_tpu = any( - tpu_nodeset is not None - for part in lkp.cfg.partitions.values() - for tpu_nodeset in part.partition_nodeset_tpu - ) - - any_dynamic = any(bool(p.partition_feature) for p in lkp.cfg.partitions.values()) - comma_params = { - "LaunchParameters": [ - "enable_nss_slurm", - "use_interactive_step", - ], - "SlurmctldParameters": [ - "cloud_reg_addrs" if any_dynamic or any_tpu else "cloud_dns", - "enable_configless", - "idle_on_node_suspend", - ], - "GresTypes": [ - "gpu" if any_gpus else None, - ], - } - - scripts_dir = lkp.cfg.install_dir or dirs.scripts - prolog_path = Path(dirs.custom_scripts / "prolog.d") - epilog_path = Path(dirs.custom_scripts / "epilog.d") - task_prolog_path = Path(dirs.custom_scripts / "task_prolog.d") - task_epilog_path = Path(dirs.custom_scripts / "task_epilog.d") - default_tree_width = 65533 if any_dynamic else 128 - - conf_options = { - **(comma_params if not no_comma_params else {}), - "Prolog": f"{prolog_path}/*" if lkp.cfg.prolog_scripts else None, - "Epilog": f"{epilog_path}/*" if lkp.cfg.epilog_scripts else None, - "TaskProlog": f"{task_prolog_path}/task-prolog" if lkp.cfg.task_prolog_scripts else None, - "TaskEpilog": f"{task_epilog_path}/task-epilog" if lkp.cfg.task_epilog_scripts else None, - "PrivateData": get("private_data", []), - "SchedulerParameters": get("scheduler_parameters", [ - "bf_continue", - "salloc_wait_nodes", - "ignore_prefer_validation", - ]), - "ResumeProgram": f"{scripts_dir}/resume_wrapper.sh", - "ResumeFailProgram": f"{scripts_dir}/suspend_wrapper.sh", - "ResumeRate": get("resume_rate", 0), - "ResumeTimeout": get("resume_timeout", 300), - "SuspendProgram": f"{scripts_dir}/suspend_wrapper.sh", - "SuspendRate": get("suspend_rate", 0), - "SuspendTimeout": get("suspend_timeout", 300), - "TreeWidth": get("tree_width", default_tree_width), - "JobSubmitPlugins": "lua" if any_tpu else None, - "TopologyPlugin": topology_plugin(lkp), - "TopologyParam": get("topology_param", "SwitchAsNodeRank"), - } - return dict_to_conf(conf_options, delim="\n") - - - - -def nodeset_lines(nodeset, lkp: util.Lookup) -> str: - template_info = lkp.template_info(nodeset.instance_template) - machine_conf = lkp.template_machine_conf(nodeset.instance_template) - - # follow https://slurm.schedmd.com/slurm.conf.html#OPT_Boards - # by setting Boards, SocketsPerBoard, CoresPerSocket, and ThreadsPerCore - gres = f"gpu:{template_info.gpu.count}" if template_info.gpu else None - node_conf = { - "RealMemory": machine_conf.memory, - "Boards": machine_conf.boards, - "SocketsPerBoard": machine_conf.sockets_per_board, - "CoresPerSocket": machine_conf.cores_per_socket, - "ThreadsPerCore": machine_conf.threads_per_core, - "CPUs": machine_conf.cpus, - "Gres": gres, - **nodeset.node_conf, - } - nodelist = lkp.nodelist(nodeset) - - return "\n".join( - map( - dict_to_conf, - [ - {"NodeName": nodelist, "State": "CLOUD", **node_conf}, - {"NodeSet": nodeset.nodeset_name, "Nodes": nodelist}, - ], - ) - ) - - -def nodeset_tpu_lines(nodeset, lkp: util.Lookup) -> str: - nodelist = lkp.nodelist(nodeset) - return "\n".join( - map( - dict_to_conf, - [ - {"NodeName": nodelist, "State": "CLOUD", **nodeset.node_conf}, - {"NodeSet": nodeset.nodeset_name, "Nodes": nodelist}, - ], - ) - ) - - -def nodeset_dyn_lines(nodeset): - """generate slurm NodeSet definition for dynamic nodeset""" - return dict_to_conf( - {"NodeSet": nodeset.nodeset_name, "Feature": nodeset.nodeset_feature} - ) - - -def partitionlines(partition, lkp: util.Lookup) -> str: - """Make a partition line for the slurm.conf""" - MIN_MEM_PER_CPU = 100 - - def defmempercpu(nodeset_name: str) -> int: - nodeset = lkp.cfg.nodeset.get(nodeset_name) - template = nodeset.instance_template - machine = lkp.template_machine_conf(template) - mem_spec_limit = int(nodeset.node_conf.get("MemSpecLimit", 0)) - return max(MIN_MEM_PER_CPU, (machine.memory - mem_spec_limit) // machine.cpus) - - defmem = min( - map(defmempercpu, partition.partition_nodeset), default=MIN_MEM_PER_CPU - ) - - nodesets = list( - chain( - partition.partition_nodeset, - partition.partition_nodeset_dyn, - partition.partition_nodeset_tpu, - ) - ) - - is_tpu = len(partition.partition_nodeset_tpu) > 0 - is_dyn = len(partition.partition_nodeset_dyn) > 0 - - oversub_exlusive = partition.enable_job_exclusive or is_tpu - power_down_on_idle = partition.enable_job_exclusive and not is_dyn - - line_elements = { - "PartitionName": partition.partition_name, - "Nodes": ",".join(nodesets), - "State": "UP", - "DefMemPerCPU": defmem, - "SuspendTime": 300, - "Oversubscribe": "Exclusive" if oversub_exlusive else None, - "PowerDownOnIdle": "YES" if power_down_on_idle else None, - **partition.partition_conf, - } - - return dict_to_conf(line_elements) - - -def suspend_exc_lines(lkp: util.Lookup) -> Iterable[str]: - static_nodelists = [] - for ns in lkp.power_managed_nodesets(): - if ns.node_count_static: - nodelist = lkp.nodelist_range(ns.nodeset_name, 0, ns.node_count_static) - static_nodelists.append(nodelist) - suspend_exc_nodes = {"SuspendExcNodes": static_nodelists} - - dyn_parts = [ - p.partition_name - for p in lkp.cfg.partitions.values() - if len(p.partition_nodeset_dyn) > 0 - ] - suspend_exc_parts = {"SuspendExcParts": [*dyn_parts]} - - return filter( - None, - [ - dict_to_conf(suspend_exc_nodes) if static_nodelists else None, - dict_to_conf(suspend_exc_parts), - ], - ) - - -def make_cloud_conf(lkp: util.Lookup) -> str: - """generate cloud.conf snippet""" - lines = [ - FILE_PREAMBLE, - conflines(lkp), - *(nodeset_lines(n, lkp) for n in lkp.cfg.nodeset.values()), - *(nodeset_dyn_lines(n) for n in lkp.cfg.nodeset_dyn.values()), - *(nodeset_tpu_lines(n, lkp) for n in lkp.cfg.nodeset_tpu.values()), - *(partitionlines(p, lkp) for p in lkp.cfg.partitions.values()), - *(suspend_exc_lines(lkp)), - ] - return "\n\n".join(filter(None, lines)) - - -def gen_cloud_conf(lkp: util.Lookup) -> None: - content = make_cloud_conf(lkp) - - conf_file = lkp.etc_dir / "cloud.conf" - conf_file.write_text(content) - util.chown_slurm(conf_file, mode=0o644) - - -def install_slurm_conf(lkp: util.Lookup) -> None: - """install slurm.conf""" - if lkp.cfg.ompi_version: - mpi_default = "pmi2" - else: - mpi_default = "none" - - conf_options = { - "name": lkp.cfg.slurm_cluster_name, - "control_addr": lkp.control_addr if lkp.control_addr else lkp.hostname_fqdn, - "control_host": lkp.control_host, - "accounting_storage_host": lkp.control_addr if lkp.cfg.controller_network_attachment else lkp.control_host, - "control_host_port": lkp.control_host_port, - "scripts": dirs.scripts, - "slurmlog": dirs.log, - "state_save": slurmdirs.state, - "mpi_default": mpi_default, - "auth_key": "slurm" if lkp.cfg.enable_slurm_auth else "munge", - } - - conf = lkp.cfg.slurm_conf_tpl.format(**conf_options) - - conf_file = lkp.etc_dir / "slurm.conf" - conf_file.write_text(conf) - util.chown_slurm(conf_file, mode=0o644) - - -def install_slurmdbd_conf(lkp: util.Lookup) -> None: - """install slurmdbd.conf""" - conf_options = { - "control_host": lkp.control_host, - "slurmlog": dirs.log, - "state_save": slurmdirs.state, - "db_name": "slurm_acct_db", - "db_user": "slurm", - "db_pass": '""', - "db_host": "localhost", - "db_port": "3306", - "auth_key": "slurm" if lkp.cfg.enable_slurm_auth else "munge", - } - - if lkp.cfg.cloudsql_secret: - secret_name = f"{lkp.cfg.slurm_cluster_name}-slurm-secret-cloudsql" - payload = json.loads(util.access_secret_version(lkp.project, secret_name)) - - if payload["db_name"] and payload["db_name"] != "": - conf_options["db_name"] = payload["db_name"] - if payload["user"] and payload["user"] != "": - conf_options["db_user"] = payload["user"] - if payload["password"] and payload["password"] != "": - conf_options["db_pass"] = payload["password"] - - db_host_str = payload["server_ip"].split(":") - if db_host_str[0]: - conf_options["db_host"] = db_host_str[0] - conf_options["db_port"] = ( - db_host_str[1] if len(db_host_str) >= 2 else "3306" - ) - - conf = lkp.cfg.slurmdbd_conf_tpl.format(**conf_options) - - conf_file = lkp.etc_dir / "slurmdbd.conf" - conf_file.write_text(conf) - util.chown_slurm(conf_file, 0o600) - - -def install_cgroup_conf(lkp: util.Lookup) -> None: - """install cgroup.conf""" - conf_file = lkp.etc_dir / "cgroup.conf" - conf_file.write_text(lkp.cfg.cgroup_conf_tpl) - util.chown_slurm(conf_file, mode=0o600) - - -def install_jobsubmit_lua(lkp: util.Lookup) -> None: - """install job_submit.lua if there are tpu nodes in the cluster""" - if not any( - tpu_nodeset is not None - for part in lkp.cfg.partitions.values() - for tpu_nodeset in part.partition_nodeset_tpu - ): - return # No TPU partitions, no need for job_submit.lua - - scripts_dir = lkp.cfg.slurm_scripts_dir or dirs.scripts - tpl = (scripts_dir / "job_submit.lua.tpl").read_text() - conf = tpl.format(scripts_dir=scripts_dir) - - conf_file = lkp.etc_dir / "job_submit.lua" - conf_file.write_text(conf) - util.chown_slurm(conf_file, 0o600) - - -def gen_cloud_gres_conf(lkp: util.Lookup) -> None: - """generate cloud_gres.conf""" - - gpu_nodes = defaultdict(list) - for nodeset in lkp.cfg.nodeset.values(): - ti = lkp.template_info(nodeset.instance_template) - gpu_count = ti.gpu.count if ti.gpu else 0 - if gpu_count: - gpu_nodes[gpu_count].append(lkp.nodelist(nodeset)) - - lines = [ - dict_to_conf( - { - "NodeName": names, - "Name": "gpu", - "File": "/dev/nvidia{}".format(f"[0-{i-1}]" if i > 1 else "0"), - } - ) - for i, names in gpu_nodes.items() - ] - lines.append("\n") - content = FILE_PREAMBLE + "\n".join(lines) - - conf_file = lkp.etc_dir / "cloud_gres.conf" - conf_file.write_text(content) - util.chown_slurm(conf_file, mode=0o600) - - -def install_gres_conf(lkp: util.Lookup) -> None: - conf_file = lkp.etc_dir / "cloud_gres.conf" - gres_conf = lkp.etc_dir / "gres.conf" - if not gres_conf.exists(): - gres_conf.symlink_to(conf_file) - util.chown_slurm(gres_conf, mode=0o600) - - -class Switch: - """ - Represents a switch in the topology.conf file. - NOTE: It's class user job to make sure that there is no leaf-less Switches in the tree - """ - - def __init__( - self, - name: str, - nodes: Optional[Iterable[str]] = None, - switches: Optional[Dict[str, "Switch"]] = None, - ): - self.name = name - self.nodes = nodes or [] - self.switches = switches or {} - - def conf_line(self) -> str: - d = {"SwitchName": self.name} - if self.nodes: - d["Nodes"] = util.to_hostlist(self.nodes) - if self.switches: - d["Switches"] = util.to_hostlist(self.switches.keys()) - return dict_to_conf(d) - - def render_conf_lines(self) -> Iterable[str]: - yield self.conf_line() - for s in sorted(self.switches.values(), key=lambda s: s.name): - yield from s.render_conf_lines() - -class TopologySummary: - """ - Represents a summary of the topology, to make judgements about changes. - To be stored in JSON file along side of topology.conf to simplify parsing. - """ - def __init__( - self, - physical_host: Optional[Dict[str, str]] = None, - down_nodes: Optional[Iterable[str]] = None, - tpu_nodes: Optional[Iterable[str]] = None, - ) -> None: - self.physical_host = physical_host or {} - self.down_nodes = set(down_nodes or []) - self.tpu_nodes = set(tpu_nodes or []) - - - @classmethod - def path(cls, lkp: util.Lookup) -> Path: - return lkp.etc_dir / "cloud_topology.summary.json" - - @classmethod - def loads(cls, s: str) -> "TopologySummary": - d = json.loads(s) - return cls( - physical_host=d.get("physical_host"), - down_nodes=d.get("down_nodes"), - tpu_nodes=d.get("tpu_nodes"), - ) - - @classmethod - def load(cls, lkp: util.Lookup) -> "TopologySummary": - p = cls.path(lkp) - if not p.exists(): - return cls() # Return empty instance - return cls.loads(p.read_text()) - - def dumps(self) -> str: - return json.dumps( - { - "physical_host": self.physical_host, - "down_nodes": list(self.down_nodes), - "tpu_nodes": list(self.tpu_nodes), - }, - indent=2) - - def dump(self, lkp: util.Lookup) -> None: - TopologySummary.path(lkp).write_text(self.dumps()) - - def _nodenames(self) -> Set[str]: - return set(self.physical_host) | self.down_nodes | self.tpu_nodes - - def requires_reconfigure(self, prev: "TopologySummary") -> bool: - """ - Reconfigure IFF one of the following occurs: - * A node is added - * A node get a non-empty physicalHost - """ - if len(self._nodenames() - prev._nodenames()) > 0: - return True - for n, ph in self.physical_host.items(): - if ph and ph != prev.physical_host.get(n): - return True - return False - -class TopologyBuilder: - def __init__(self) -> None: - self._r = Switch("") # fake root, not part of the tree - self.summary = TopologySummary() - - def add(self, path: List[str], nodes: Iterable[str]) -> None: - n = self._r - assert path - for p in path: - n = n.switches.setdefault(p, Switch(p)) - n.nodes = [*n.nodes, *nodes] - - def render_conf_lines(self) -> Iterable[str]: - if not self._r.switches: - return [] # type: ignore - for s in sorted(self._r.switches.values(), key=lambda s: s.name): - yield from s.render_conf_lines() - - def compress(self) -> "TopologyBuilder": - compressed = TopologyBuilder() - compressed.summary = self.summary - def _walk( - u: Switch, c: Switch - ): # u: uncompressed node, c: its counterpart in compressed tree - pref = f"{c.name}_" if c != compressed._r else "s" - for i, us in enumerate(sorted(u.switches.values(), key=lambda s: s.name)): - cs = Switch(f"{pref}{i}", nodes=us.nodes) - c.switches[cs.name] = cs - _walk(us, cs) - - _walk(self._r, compressed._r) - return compressed - - -def add_tpu_nodeset_topology(nodeset: NSDict, bldr: TopologyBuilder, lkp: util.Lookup): - tpuobj = tpu.TPU.make(nodeset.nodeset_name, lkp) - static, dynamic = lkp.nodenames(nodeset) - - pref = ["tpu-root", f"ns_{nodeset.nodeset_name}"] - if tpuobj.vmcount == 1: # Put all nodes in one switch - all_nodes = list(chain(static, dynamic)) - bldr.add(pref, all_nodes) - bldr.summary.tpu_nodes.update(all_nodes) - return - - # Chunk nodes into sub-switches of size `vmcount` - chunk_num = 0 - for nodenames in (static, dynamic): - for nodeschunk in util.chunked(nodenames, n=tpuobj.vmcount): - chunk_name = f"{nodeset.nodeset_name}-{chunk_num}" - chunk_num += 1 - bldr.add([*pref, chunk_name], nodeschunk) - bldr.summary.tpu_nodes.update(nodeschunk) - -_SLURM_TOPO_ROOT = "slurm-root" - -def _make_physical_path(physical_host: str) -> List[str]: - assert physical_host.startswith("/"), f"Unexpected physicalHost: {physical_host}" - parts = physical_host[1:].split("/") - # Due to issues with Slurm's topology plugin, we can not use all components of `physicalHost`, - # trim it down to `cluster/rack`. - short_path = parts[:2] - return [_SLURM_TOPO_ROOT, *short_path] - -def add_nodeset_topology( - nodeset: NSDict, bldr: TopologyBuilder, lkp: util.Lookup -) -> None: - up_nodes = set() - default_path = [_SLURM_TOPO_ROOT, f"ns_{nodeset.nodeset_name}"] - - for inst in lkp.instances().values(): - try: - if lkp.node_nodeset_name(inst.name) != nodeset.nodeset_name: - continue - except Exception: - continue - - phys_host = inst.resource_status.physical_host or "" - bldr.summary.physical_host[inst.name] = phys_host - up_nodes.add(inst.name) - - if phys_host: - bldr.add(_make_physical_path(phys_host), [inst.name]) - else: - bldr.add(default_path, [inst.name]) - - down_nodes = [] - for node in chain(*lkp.nodenames(nodeset)): - if node not in up_nodes: - down_nodes.append(node) - if down_nodes: - bldr.add(default_path, down_nodes) - bldr.summary.down_nodes.update(down_nodes) - -def gen_topology(lkp: util.Lookup) -> TopologyBuilder: - bldr = TopologyBuilder() - for ns in lkp.cfg.nodeset_tpu.values(): - add_tpu_nodeset_topology(ns, bldr, lkp) - for ns in lkp.cfg.nodeset.values(): - add_nodeset_topology(ns, bldr, lkp) - return bldr - -def gen_topology_conf(lkp: util.Lookup) -> Tuple[bool, TopologySummary]: - """ - Generates slurm topology.conf. - Returns whether the topology.conf got updated. - """ - topo = gen_topology(lkp).compress() - conf_file = lkp.etc_dir / "cloud_topology.conf" - - with open(conf_file, "w") as f: - f.writelines(FILE_PREAMBLE + "\n") - for line in topo.render_conf_lines(): - f.write(line) - f.write("\n") - f.write("\n") - - prev_summary = TopologySummary.load(lkp) - return topo.summary.requires_reconfigure(prev_summary), topo.summary - -def install_topology_conf(lkp: util.Lookup) -> None: - conf_file = lkp.etc_dir / "cloud_topology.conf" - summary_file = lkp.etc_dir / "cloud_topology.summary.json" - topo_conf = lkp.etc_dir / "topology.conf" - - if not topo_conf.exists(): - topo_conf.symlink_to(conf_file) - - util.chown_slurm(conf_file, mode=0o600) - util.chown_slurm(summary_file, mode=0o600) - - -def gen_controller_configs(lkp: util.Lookup) -> None: - install_slurm_conf(lkp) - install_slurmdbd_conf(lkp) - gen_cloud_conf(lkp) - gen_cloud_gres_conf(lkp) - install_gres_conf(lkp) - install_cgroup_conf(lkp) - install_jobsubmit_lua(lkp) - - if topology_plugin(lkp) == TOPOLOGY_PLUGIN_TREE: - _, summary = gen_topology_conf(lkp) - summary.dump(lkp) - install_topology_conf(lkp) diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py deleted file mode 100644 index cd2e41e5af..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/file_cache.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Any -from pathlib import Path -import shutil -import pickle - -import logging -log = logging.getLogger() - -# Can't reuse tool from util.py to avoid circular dependencies -# TODO: break down util.py for better modularity. -def _chown_slurm(path: Path) -> None: - shutil.chown(path, user="slurm", group="slurm") - -class FileCache: - def __init__(self, path: Path): - self.path = path - - def get(self, key: str) -> Any | None: - p = self.path / key - if not p.exists(): - return None - - try: - with p.open("rb") as f: - return pickle.load(f) - - except Exception as e: - log.warning(f"Failed to read cached value at {p}: {e}") - return None - - def set(self, key: str, data: Any) -> None: - p = self.path / key - - try: - # Create & chown before writing to minimize chances - # of ending up with root-owned corrupted file that can't be cleaned up - # TODO: restrict usage of cache by root to avoid all this complexity - # or have a cache per user. - p.touch(exist_ok=True) - _chown_slurm(p) - with p.open("wb") as f: - pickle.dump(data, f) - - except Exception as e: - log.warning(f"Failed to write cached value at {p}: {e}") - - -class NoCache: - def get(self, key: str) -> Any: - log.warning("No cache used") - return None - - def set(self, key: str, data: Any) -> None: - log.warning("No cache used") - - -def cache(name: str) -> FileCache | NoCache: - try: - path = Path("/tmp/slurm_gcp_cache/") / name - if not path.exists(): - path.mkdir(exist_ok=True, parents=True) - _chown_slurm(path) - return FileCache(path) - except: - log.exception(f"Failed to create cache, fallback to NoCache") - return NoCache() diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py deleted file mode 100644 index df0fd8ebe0..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/get_tpu_vmcount.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright 2024 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import util -import tpu - - -def get_vmcount_of_tpu_part(part): - res = 0 - lkp = util.lookup() - for ns in lkp.cfg.partitions[part].partition_nodeset_tpu: - tpu_obj = tpu.TPU.make(ns, lkp) - if res == 0: - res = tpu_obj.vmcount - else: - if res != tpu_obj.vmcount: - # this should not happen, that in the same partition there are different vmcount nodesets - return -1 - return res - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--partitions", - "-p", - help="The partition(s) to retrieve the TPU vmcount value for.", - ) - args = parser.parse_args() - if not args.partitions: - exit(0) - - # useful exit code - # partition does not exists in config.yaml, thus do not exist in slurm - PART_INVALID = -1 - # in the same partition there are nodesets with different vmcounts - DIFF_VMCOUNTS_SAME_PART = -2 - # partition is a list of partitions in which at least two of them have different vmcount - DIFF_PART_DIFFERENT_VMCOUNTS = -3 - vmcounts = [] - # valid equals to 0 means that we are ok, otherwise it will be set to one of the previously defined exit codes - valid = 0 - for part in args.partitions.split(","): - if part not in util.lookup().cfg.partitions: - valid = PART_INVALID - break - else: - if util.lookup().partition_is_tpu(part): - vmcount = get_vmcount_of_tpu_part(part) - if vmcount == -1: - valid = DIFF_VMCOUNTS_SAME_PART - break - vmcounts.append(vmcount) - else: - vmcounts.append(0) - # this means that there are different vmcounts for these partitions - if valid == 0 and len(set(vmcounts)) != 1: - valid = DIFF_PART_DIFFERENT_VMCOUNTS - if valid != 0: - print(f"VMCOUNT:{valid}") - else: - print(f"VMCOUNT:{vmcounts[0]}") diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl deleted file mode 100644 index 810a0742b0..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/job_submit.lua.tpl +++ /dev/null @@ -1,103 +0,0 @@ -SCRIPTS_DIR = "{scripts_dir}" -NO_VAL = 4294967294 --- get_tpu_vmcount.py exit code -PART_INVALID = -1 -- partition does not exists in config.yaml, thus do not exist in slurm -DIFF_VMCOUNTS_SAME_PART = -2 -- in the same partition there are nodesets with different vmcounts -DIFF_PART_DIFFERENT_VMCOUNTS = -3 -- partition is a list of partitions in which at least two of them have different vmcount -UNKWOWN_ERROR = -4 -- get_tpu_vmcount.py did not return a valid response - -function get_part(job_desc, part_list) - if job_desc.partition then - return job_desc.partition - end - for name, val in pairs(part_list) do - if val.flag_default == 1 then - return name - end - end - return nil -end - -function os.capture(cmd, raw) - local handle = assert(io.popen(cmd, 'r')) - local output = assert(handle:read('*a')) - handle:close() - return output -end - -function get_vmcount(part) - local cmd = SCRIPTS_DIR .. "/get_tpu_vmcount.py -p " .. part - local out = os.capture(cmd, true) - for line in out:gmatch("(.-)\r?\n") do - local tag, val = line:match("([^:]+):([^:]+)") - if tag == "VMCOUNT" then - return tonumber(val) - end - end - return UNKWOWN_ERROR -end - -function slurm_job_submit(job_desc, part_list, submit_uid) - local part = get_part(job_desc, part_list) - local vmcount = get_vmcount(part) - -- Only do something if the job is in a TPU partition, if vmcount is 0, it implies that the partition(s) specified are not TPU ones - if vmcount == 0 then - return slurm.SUCCESS - end - -- This is a TPU job, but as the vmcount is 1 it can he handled the same way - if vmcount == 1 then - return slurm.SUCCESS - end - -- Check for errors - if vmcount == PART_INVALID then - slurm.log_user("Invalid partition specified " .. part) - return slurm.FAILURE - end - if vmcount == DIFF_VMCOUNTS_SAME_PART then - slurm.log_user("In partition(s) " .. part .. - " there are more than one tpu nodeset vmcount, this should not happen.") - return slurm.ERROR - end - if vmcount == DIFF_PART_DIFFERENT_VMCOUNTS then - slurm.log_user("In partition list " .. part .. - " there are more than one TPU types, cannot determine which is the correct vmcount to use, please retry with only one partition.") - return slurm.FAILURE - end - if vmcount == UNKWOWN_ERROR then - slurm.log_user("Something went wrong while executing get_tpu_vmcount.py.") - return slurm.ERROR - end - -- This is surely a TPU node - if vmcount > 1 then - local min_nodes = job_desc.min_nodes - local max_nodes = job_desc.max_nodes - -- if not specified assume it is one, this should be improved taking into account the cpus, mem, and other factors - if min_nodes == NO_VAL then - min_nodes = 1 - max_nodes = 1 - end - -- as max_nodes can be higher than the nodes in the partition, we are not able to calculate with certainty the nodes that this job will have if this value is set to something - -- different than min_nodes - if min_nodes ~= max_nodes then - slurm.log_user("Max nodes cannot be set different than min nodes for the TPU partitions.") - return slurm.ERROR - end - -- Set the number of switches to the number of nodes originally requested by the job, as the job requests "TPU groups" - job_desc.req_switch = min_nodes - - -- Apply the node increase into the job description. - job_desc.min_nodes = min_nodes * vmcount - job_desc.max_nodes = max_nodes * vmcount - -- if job_desc.features then - -- slurm.log_user("Features: %s",job_desc.features) - -- end - end - - return slurm.SUCCESS -end - -function slurm_job_modify(job_desc, job_rec, part_list, modify_uid) - return slurm.SUCCESS -end - -return slurm.SUCCESS diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py deleted file mode 100644 index cabd6e3e9f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/load_bq.py +++ /dev/null @@ -1,352 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Dict, Callable, Any -import argparse -import os -import shelve -import uuid -from collections import namedtuple -from datetime import datetime, timedelta, timezone -from pathlib import Path -from pprint import pprint - -import util -from google.api_core import exceptions, retry -from google.cloud import bigquery as bq -from google.cloud.bigquery import SchemaField # type: ignore -from util import lookup, run - -SACCT = "sacct" -script = Path(__file__).resolve() - -DEFAULT_TIMESTAMP_FILE = script.parent / "bq_timestamp" -timestamp_file = Path(os.environ.get("TIMESTAMP_FILE", DEFAULT_TIMESTAMP_FILE)) -# The maximum request to insert_rows is 10MB, each sacct row is about 1200 bytes or ~ 8000 rows. -# Set to 5000 for a little wiggle room. -BQ_ROW_BATCH_SIZE = 5000 - -# cluster_id_file = script.parent / 'cluster_uuid' -# try: -# cluster_id = cluster_id_file.read_text().rstrip() -# except FileNotFoundError: -# cluster_id = uuid.uuid4().hex -# cluster_id_file.write_text(cluster_id) - -job_idx_cache_path = script.parent / "bq_job_idx_cache" - -SLURM_TIME_FORMAT = r"%Y-%m-%dT%H:%M:%S" - - -def make_datetime(time_string): - if time_string == "None": - return None - return datetime.strptime(time_string, SLURM_TIME_FORMAT).replace( - tzinfo=timezone.utc - ) - - -def make_time_interval(seconds): - sign = 1 - if seconds < 0: - sign = -1 - seconds = abs(seconds) - d, r = divmod(seconds, 60 * 60 * 24) - h, r = divmod(r, 60 * 60) - m, s = divmod(r, 60) - d *= sign - h *= sign - return f"{d}D {h:02}:{m:02}:{s}" - - -converters: Dict[str, Callable[[Any], Any]] = { - "DATETIME": make_datetime, - "INTERVAL": make_time_interval, - "STRING": str, - "INT64": lambda n: int(n or 0), -} - - -def schema_field(field_name, data_type, description, required=False): - return SchemaField( - field_name, - data_type, - description=description, - mode="REQUIRED" if required else "NULLABLE", - ) - - -schema_fields = [ - schema_field("cluster_name", "STRING", "cluster name", required=True), - schema_field("cluster_id", "STRING", "UUID for the cluster", required=True), - schema_field("entry_uuid", "STRING", "entry UUID for the job row", required=True), - schema_field( - "job_db_uuid", "STRING", "job db index from the slurm database", required=True - ), - schema_field("job_id_raw", "INT64", "raw job id", required=True), - schema_field("job_id", "STRING", "job id", required=True), - schema_field("state", "STRING", "final job state", required=True), - schema_field("job_name", "STRING", "job name"), - schema_field("partition", "STRING", "job partition"), - schema_field("submit_time", "DATETIME", "job submit time"), - schema_field("start_time", "DATETIME", "job start time"), - schema_field("end_time", "DATETIME", "job end time"), - schema_field("elapsed_raw", "INT64", "STRING", "job run time in seconds"), - # schema_field("elapsed_time", "INTERVAL", "STRING", "job run time interval"), - schema_field("timelimit_raw", "STRING", "job timelimit in minutes"), - schema_field("timelimit", "STRING", "job timelimit"), - # schema_field("num_tasks", "INT64", "number of allocated tasks in job"), - schema_field("nodelist", "STRING", "names of nodes allocated to job"), - schema_field("user", "STRING", "user responsible for job"), - schema_field("uid", "INT64", "uid of job user"), - schema_field("group", "STRING", "group of job user"), - schema_field("gid", "INT64", "gid of job user"), - schema_field("wckey", "STRING", "job wckey"), - schema_field("qos", "STRING", "job qos"), - schema_field("comment", "STRING", "job comment"), - schema_field("admin_comment", "STRING", "job admin comment"), - # extra will be added in 23.02 - # schema_field("extra", "STRING", "job extra field"), - schema_field("exitcode", "STRING", "job exit code"), - schema_field("alloc_cpus", "INT64", "count of allocated CPUs"), - schema_field("alloc_nodes", "INT64", "number of nodes allocated to job"), - schema_field("alloc_tres", "STRING", "allocated trackable resources (TRES)"), - # schema_field("system_cpu", "INTERVAL", "cpu time used by parent processes"), - # schema_field("cpu_time", "INTERVAL", "CPU time used (elapsed * cpu count)"), - schema_field("cpu_time_raw", "INT64", "CPU time used (elapsed * cpu count)"), - # schema_field("ave_cpu", "INT64", "Average CPU time of all tasks in job"), - # schema_field( - # "tres_usage_tot", - # "STRING", - # "Tres total usage by all tasks in job", - # ), -] - - -slurm_field_map = { - "job_db_uuid": "DBIndex", - "job_id_raw": "JobIDRaw", - "job_id": "JobID", - "state": "State", - "job_name": "JobName", - "partition": "Partition", - "submit_time": "Submit", - "start_time": "Start", - "end_time": "End", - "elapsed_raw": "ElapsedRaw", - "elapsed_time": "Elapsed", - "timelimit_raw": "TimelimitRaw", - "timelimit": "Timelimit", - "num_tasks": "NTasks", - "nodelist": "Nodelist", - "user": "User", - "uid": "Uid", - "group": "Group", - "gid": "Gid", - "wckey": "Wckey", - "qos": "Qos", - "comment": "Comment", - "admin_comment": "AdminComment", - # "extra": "Extra", - "exit_code": "ExitCode", - "alloc_cpus": "AllocCPUs", - "alloc_nodes": "AllocNodes", - "alloc_tres": "AllocTres", - "system_cpu": "SystemCPU", - "cpu_time": "CPUTime", - "cpu_time_raw": "CPUTimeRaw", - "ave_cpu": "AveCPU", - "tres_usage_tot": "TresUsageInTot", -} - -# new field name is the key for job_schema. Used to lookup the datatype when -# creating the job rows -job_schema = {field.name: field for field in schema_fields} -# Order is important here, as that is how they are parsed from sacct output -Job = namedtuple("Job", job_schema.keys()) # type: ignore -# ... see https://github.com/python/mypy/issues/848 - -client = bq.Client( - project=lookup().cfg.project, - credentials=util.default_credentials(), - client_options=util.create_client_options(util.ApiEndpoint.BQ), -) -dataset_id = f"{lookup().cfg.slurm_cluster_name}_job_data" -dataset = bq.DatasetReference(project=lookup().project, dataset_id=dataset_id) -table = bq.Table( - bq.TableReference(dataset, f"jobs_{lookup().cfg.slurm_cluster_name}"), schema_fields -) - - -class JobInsertionFailed(Exception): - pass - - -def make_job_row(job): - job_row = { - field_name: converters[field.field_type](job[field_name]) - for field_name, field in job_schema.items() - if field_name in job - } - job_row["entry_uuid"] = uuid.uuid4().hex - job_row["cluster_id"] = lookup().cfg.cluster_id - job_row["cluster_name"] = lookup().cfg.slurm_cluster_name - return job_row - - -def load_slurm_jobs(start, end): - states = ",".join( - ( - "BOOT_FAIL", - "CANCELLED", - "COMPLETED", - "DEADLINE", - "FAILED", - "NODE_FAIL", - "OUT_OF_MEMORY", - "PREEMPTED", - "REQUEUED", - "REVOKED", - "TIMEOUT", - ) - ) - start_iso = start.isoformat(timespec="seconds") - end_iso = end.isoformat(timespec="seconds") - # slurm_fields and bq_fields will be in matching order - slurm_fields = ",".join(slurm_field_map.values()) - bq_fields = slurm_field_map.keys() - cmd = ( - f"{SACCT} --start {start_iso} --end {end_iso} -X -D --format={slurm_fields} " - f"--state={states} --parsable2 --noheader --allusers --duplicates" - ) - text = run(cmd).stdout.splitlines() - # zip pairs bq_fields with the value from sacct - jobs = [dict(zip(bq_fields, line.split("|"))) for line in text] - - # The job index cache allows us to avoid sending duplicate jobs. This avoids a race condition with updating the database. - with shelve.open(str(job_idx_cache_path), flag="r") as job_idx_cache: - job_rows = [ - make_job_row(job) - for job in jobs - if str(job["job_db_uuid"]) not in job_idx_cache - ] - return job_rows - - -def init_table(): - global dataset - global table - dataset = client.create_dataset(dataset, exists_ok=True) # type: ignore - table = client.create_table(table, exists_ok=True) - until_found = retry.Retry(predicate=retry.if_exception_type(exceptions.NotFound)) - table = client.get_table(table, retry=until_found) - # cannot add required fields to an existing schema - table.schema = schema_fields - table = client.update_table(table, ["schema"]) - - -def purge_job_idx_cache(): - purge_time = datetime.now() - timedelta(minutes=30) - with shelve.open(str(job_idx_cache_path), writeback=True) as cache: - to_delete = [] - for idx, stamp in cache.items(): - if stamp < purge_time: - to_delete.append(idx) - for idx in to_delete: - del cache[idx] - - -def bq_submit(jobs): - try: - result = client.insert_rows(table, jobs) - except exceptions.NotFound as e: - print(f"failed to upload job data, table not yet found: {e}") - raise e - except Exception as e: - print(f"failed to upload job data: {e}") - raise e - if result: - pprint(jobs) - pprint(result) - raise JobInsertionFailed("failed to upload job data to big query") - print(f"successfully loaded {len(jobs)} jobs") - - -def get_time_window(): - if not timestamp_file.is_file(): - timestamp_file.touch() - try: - timestamp = datetime.strptime( - timestamp_file.read_text().rstrip(), SLURM_TIME_FORMAT - ) - # time window will overlap the previous by 10 minutes. Duplicates will be filtered out by the job_idx_cache - start = timestamp - timedelta(minutes=10) - except ValueError: - # timestamp 1 is 1 second after the epoch; timestamp 0 is special for sacct - start = datetime.fromtimestamp(1) - # end is now() truncated to the last second - end = datetime.now().replace(microsecond=0) - return start, end - - -def write_timestamp(time): - timestamp_file.write_text(time.isoformat(timespec="seconds")) - - -def update_job_idx_cache(jobs, timestamp): - with shelve.open(str(job_idx_cache_path), writeback=True) as job_idx_cache: - for job in jobs: - job_idx = str(job["job_db_uuid"]) - job_idx_cache[job_idx] = timestamp - - -def main(): - if not lookup().cfg.enable_bigquery_load: - print("bigquery load is not currently enabled") - exit(0) - init_table() - - start, end = get_time_window() - jobs = load_slurm_jobs(start, end) - # on failure, an exception will cause the timestamp not to be rewritten. So - # it will try again next time. If some writes succeed, we don't currently - # have a way to not submit duplicates next time. - if jobs: - num_batches = (len(jobs) - 1) // BQ_ROW_BATCH_SIZE + 1 - print( - f"loading {num_batches} batches of BigQuery data in batches of size : {BQ_ROW_BATCH_SIZE}" - ) - for batch_indx, job_indx in enumerate(range(0, len(jobs), BQ_ROW_BATCH_SIZE)): - print(f"loading BigQuery data batch {batch_indx} of {num_batches}") - bq_submit(jobs[job_indx : job_indx + BQ_ROW_BATCH_SIZE]) - write_timestamp(end) - update_job_idx_cache(jobs, end) - - -parser = argparse.ArgumentParser(description="submit slurm job data to big query") -parser.add_argument( - "timestamp_file", - nargs="?", - action="store", - type=Path, - help="specify timestamp file for reading and writing the time window start. Precedence over TIMESTAMP_FILE env var.", -) - -purge_job_idx_cache() -if __name__ == "__main__": - args = parser.parse_args() - if args.timestamp_file: - timestamp_file = args.timestamp_file.resolve() - main() diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py deleted file mode 100644 index d4a4477f83..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/local_pubsub.py +++ /dev/null @@ -1,196 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -""" -Implementation of message queue that mimics interface of GCP (PubSub)[https://cloud.google.com/pubsub] - -Messages are stored on controller state disk (to survive controller re-creation) with following layout: - -// -ā”œ- -| ā””- -ā””- .staging - ā””- - ā””- - -One message is one immutable file, that will be deleted after acknowledgement. -NOTE: Implementation assumes that both `` and `.staging/` are on the same disk device, -so it can rely on atomic "move / rename" operation. -""" -from typing import Any -import util -import json -from dataclasses import dataclass -from datetime import datetime -from pathlib import Path -import os -import uuid - -import logging -log = logging.getLogger() - - -@dataclass(frozen=True) -class Message: - id: str - created: datetime - data: Any - - def to_json(self) -> dict[str, str]: - return dict( - id=self.id, - created=self.created.isoformat(), - data=self.data) - - @classmethod - def from_json(cls, data: dict[str, str]) -> 'Message': - return cls( - id=data['id'], - created=datetime.fromisoformat(data['created']), - data=data['data']) - -class Topic: - """ - Acts as PubSub topic (https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics). - We can have multiple instances of - """ - def __init__(self, path: Path, staging: Path) -> None: - self._path = path - self._staging = staging - - def _gen_id(self, created: datetime) -> str: - ts = created.strftime("%Y_%m_%d-%H_%M_%S") - suf = str(uuid.uuid4())[:8] - return f"{ts}-{suf}" - - def publish(self, data: Any) -> None: - created = util.now() - id = self._gen_id(created) - msg = Message(id=id, created=created, data=data) - - staged = self._staging / msg.id - dst = self._path / msg.id - - # Write to stagin area first then perform atomic move - # to prevent "reads of partial writes" - staged.write_text(json.dumps(msg.to_json())) - util.chown_slurm(staged) - staged.rename(dst) - - -class Subscription: - """ - Acts as PubSub subscription (https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions) - with following settings: - - ``` - ackDeadlineSeconds = +Inf # don't resend message that was already being delivered but not acked yet - retainAckedMessages = False # don't persist messages that were already acked - enableMessageOrdering = True # delivers messages in chronoligical order - messageRetentionDuration = +Inf # don't expire messages - deadLetterPolicy = None # "deadlettering" is disabled, subscriber should take care of any poisonous messages - retryPolicy = { # NACKed message will be re-delievered after some time - minimumBackoff = 30s # NOTE: Practically there is no timer, but Subscription instance will not try to re-deliver NACKed messages. - maximumBackoff = 30s # Assumes that slurmsync runs every 30+ sec. - } - ``` - - IMPORTANT: Should only be run as part of slurmsync, - this is our way to ensure that at most one instance exists at a time. - There is no concurancy safeguards in place, avoid multithreaded `pull`, - while multithreaded `ack` & `modify_ack_deadline` are OK. - """ - - def __init__(self, path: Path) -> None: - self._path: Path = path - # contains ALL messages pulled by this subscription instance - # both acked, nacked, and still being processed - # used to prevent double delivery within lifetime of subscription (slurmsync) - self._pulled: set[str] = set() - - def _delete(self, id: str) -> None: - log.debug(f"removing {id}") - try: - os.unlink(self._path / id) - except: - log.exception(f"Failed to remove message {id}") - - def _read_msg(self, id: str) -> Message | None: - try: - with open(self._path / id, 'r') as f: - content = json.loads(f.read()) - return Message.from_json(content) - except Exception: - log.exception(f"Failed to read message {id}") - self._delete(id) # delete message to reduce "deadlettering" - return None - - def pull(self, max_messages: int) -> list[Message]: - if not self._path.exists(): - log.warning(f"Topic {self._path} does not exist") - return [] - res = [] - ls = sorted(os.listdir(self._path)) - for name in ls: - msg = self._read_msg(name) - if msg is not None and msg.id not in self._pulled: - self._pulled.add(msg.id) - res.append(msg) - - if len(res) >= max_messages: - break - return res - - - def ack(self, ids: list[str]) -> None: - for id in ids: - self._delete(id) - - - def modify_ack_deadline(self, ids: list[str], deadline: int) -> None: - """ - Modifies the ack deadline for a specific message. - IMPORTANT: Only accepts deadline=0, which is a way to NACK - Any other values are also meaningless due to ackDeadlineSeconds==+Inf - """ - assert deadline == 0 # no op, next subscriber (slurmsync) will pick this up - - -# Topics and Subscriptions are singletons -# TODO: consider making thread-safe -_topics = {} -_subscriptions = {} - -def _make_path(name: str) -> Path: - p = util.slurmdirs.state / "pubsub" / name - p.mkdir(parents=True, exist_ok=True) - util.chown_slurm(p) - return p - -def _make_staging_path(name: str) -> Path: - p = util.slurmdirs.state / "pubsub" / ".staging" / name - p.mkdir(parents=True, exist_ok=True) - util.chown_slurm(p) - return p - -def topic(name: str) -> Topic: - if name not in _topics: - _topics[name] = Topic(_make_path(name), _make_staging_path(name)) - return _topics[name] - -def subscription(name: str) -> Subscription: - if name not in _subscriptions: - _subscriptions[name] = Subscription(_make_path(name)) - return _subscriptions[name] diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py deleted file mode 100644 index 35eacc05e0..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/mig_flex.py +++ /dev/null @@ -1,214 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Optional - -import util -import uuid -from addict import Dict as NSDict # type: ignore -from datetime import timedelta -from collections import defaultdict -import logging -from time import sleep - -log = logging.getLogger() - - -def _duration(flex_options: NSDict, job_id: Optional[int], lkp: util.Lookup) -> int: - dur = flex_options.max_run_duration - if not job_id or not flex_options.use_job_duration: - return dur - - job = lkp.job(job_id) - if not job or not job.duration: - return dur - - if timedelta(minutes=10) <= job.duration <= timedelta(weeks=1): - return int(job.duration.total_seconds()) - - log.info("Job TimeLimit cannot be less than 10 minutes or exceed one week") - return dur - - -def resume_flex_chunk(nodes: List[str], job_id: Optional[int], lkp: util.Lookup) -> None: - assert nodes - model = nodes[0] - nodeset = lkp.node_nodeset(model) - zones = nodeset.zone_policy_allow - assert len(zones) == 1 - zone = zones[0] - - assert nodeset.dws_flex.enabled - - uid = str(uuid.uuid4())[:8] - if job_id: - mig_name = f"{lkp.cfg.slurm_cluster_name}-{nodeset.nodeset_name}-job-{job_id}-{uid}" - else: - mig_name = f"{lkp.cfg.slurm_cluster_name}-{nodeset.nodeset_name}-{uid}" - - # Create MIG - req = lkp.compute.instanceGroupManagers().insert( - project=lkp.project, - zone=zone, - body=dict( - name=mig_name, - versions=[dict( - instanceTemplate=nodeset.instance_template)], - targetSize=0, - # TODO(FLEX): uncomment once moved to RMIG - # distributionPolicy=dict( - # zones=[ - # dict(zone=f"zones/{z}") for z in nodeset.zone_policy_allow - # ], - # targetShape="ANY_SINGLE_ZONE" ), - #updatePolicy = dict( - # instanceRedistributionType = "NONE" ), - instanceLifecyclePolicy=dict( - defaultActionOnFailure= "DO_NOTHING" ), # TODO(FLEX): Not supported yet, migrate once supported - ) - ) - util.log_api_request(req) - op = req.execute() - res = util.wait_for_operation(op) - assert "error" not in res, f"{res}" - - # Create resize request - req = lkp.compute.instanceGroupManagerResizeRequests().insert( - project=lkp.project, - zone=zone, - instanceGroupManager=mig_name, - body=dict( - name="initial-resize", - instances=[dict(name=n) for n in nodes], - requested_run_duration=dict( - seconds=_duration(nodeset.dws_flex, job_id, lkp) - ) - ) - ) - util.log_api_request(req) - op = req.execute() - res = util.wait_for_operation(op) - assert "error" not in res, f"{res}" - -def _suspend_flex_mig(mig_self_link: str, nodes: List[str], lkp: util.Lookup) -> None: - assert nodes - model = nodes[0] - nodeset = lkp.node_nodeset(model) - zones = nodeset.zone_policy_allow - assert len(zones) == 1 - zone = zones[0] - project=lkp.project - instanceGroupManager=util.trim_self_link(mig_self_link) - - links = [ - f"zones/{inst.zone}/instances/{inst.name}" - for inst in [ - lkp.instance(node) for node in nodes - ] if inst - ] - - target_mig=lkp.get_mig(lkp.project, zone, instanceGroupManager) - assert target_mig - - # TODO(FLEX): This will not work if MIG didn't obtain capacity yet. - # The request will fail and MIG will continue provisioning. - # Instead whole MIG should be deleted. - # + All other instances in MIG are not provisioned also, safe to delete - # - Need to come up will clear test to differentiate non-provisioned MIG and single VM being down; - # Particularly CRITICAL due to ActionOnFailure=DO_NOTHING - # - Need to `down_nodes_notify_jobs` for all nodes in MIG, make sure that it doesn't interfere with Slurm suspend-flow. - - if target_mig["targetSize"] == len(nodes): #We can just delete the whole MIG in this case - req = lkp.compute.instanceGroupManagers().delete( - project=project, - zone=zone, - instanceGroupManager=instanceGroupManager, - ) - else: - req = lkp.compute.instanceGroupManagers().deleteInstances( - project=project, - zone=zone, - instanceGroupManager=instanceGroupManager, - body=dict( - instances=links, - skipInstancesOnValidationError=True, - ) - ) - - util.log_api_request(req) - op = req.execute() - - res = util.wait_for_operation(op) - assert "error" not in res, f"{res}" - -def _suspend_provisioning_inst(nodes:List[str], node_template:str, lkp: util.Lookup) -> None: - assert nodes - model = nodes[0] - nodeset = lkp.node_nodeset(model) - zones = nodeset.zone_policy_allow - assert len(zones) == 1 - zone = zones[0] - - mig_list=lkp.get_mig_list(lkp.project, zone) #Validated via terraform that this is one - - # FLEX (#TODO): If we enter this conditional it's likely this was called so early that MIG creation hasn't started - # Consider potentially retrying? No natural mechanism for retry currently but we could - # perhaps use slurmsync and then try it again to ensure it wasn't a case of being too early. - # This is important since we're now enabling long ResumeTimeout (Slurm won't call suspend on node within reasonable timeframe) - # so until we do this is slurmsync this is a temporary workaround. - - if not mig_list or not mig_list.get("items"): - log.info("No matching MIG found to delete!") - sleep(5) - mig_list=lkp.get_mig_list(lkp.project, zone) - if not mig_list or not mig_list.get("items"): - return - - for mig in mig_list["items"]: - if mig["instanceTemplate"] == node_template: - if mig["currentActions"]["creating"] > 0 and mig["targetSize"] == mig["currentActions"]["creating"]: - req = lkp.compute.instanceGroupManagers().delete( - project=lkp.project, - zone=zone, - instanceGroupManager=util.trim_self_link(mig["selfLink"]), - ) - - util.log_api_request(req) - op = req.execute() - - res = util.wait_for_operation(op) - assert "error" not in res, f"{res}" - return - - log.info("No matching MIG found to delete!") - -def suspend_flex_nodes(nodes: List[str], lkp: util.Lookup) -> None: - by_mig = defaultdict(list) - not_provisioned = defaultdict(list) - for node in nodes: - inst = lkp.instance(node) - if not inst: - not_provisioned[lkp.node_template(node)].append(node) - else: - mig = inst.metadata.get("created-by") - if not mig: - log.error(f"Can not suspend {node}, can not find associated MIG") - continue - by_mig[mig].append(node) - - for mig, nodes in by_mig.items(): - _suspend_flex_mig(mig, nodes, lkp) - - for node_template, nodes in not_provisioned.items(): - _suspend_provisioning_inst(nodes, node_template, lkp) diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt deleted file mode 100644 index 2ab3162ccf..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements-dev.txt +++ /dev/null @@ -1,9 +0,0 @@ -pytest -pytest-mock -pytest_unordered -mock - -types-mock -types-httplib2 -types-requests -types-PyYAML diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt deleted file mode 100644 index d00e0cf3a3..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -addict==2.4.0 -google-api-core==2.19.0 -google-api-python-client==2.93.0 -google-auth==2.22.0 -google-auth-httplib2==0.1.0 -google-cloud-bigquery==3.11.3 -google-cloud-core==2.3.3 -google-cloud-secret-manager~=2.22 -google-cloud-storage==2.10.0 -google-cloud-tpu==1.10.0 -google-resumable-media==2.5.0 -googleapis-common-protos==1.59.1 -grpcio==1.56.2 -grpcio-status==1.56.2 -httplib2==0.22.0 -more-executors==2.11.4 -pyyaml==6.0.2 -requests==2.32.4 diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py deleted file mode 100644 index 966a6bb852..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume.py +++ /dev/null @@ -1,664 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Optional, Dict, Any -import argparse -from datetime import timedelta -import shlex -import json -import logging -import os -import yaml -import collections -from pathlib import Path -from dataclasses import dataclass -from addict import Dict as NSDict # type: ignore - -import util -from util import ( - chunked, - ensure_execute, - execute_with_futures, - log_api_request, - map_with_futures, - run, - separate, - to_hostlist, - trim_self_link, - wait_for_operation, -) -from util import lookup, ReservationDetails -import tpu -import mig_flex - -log = logging.getLogger() - -PLACEMENT_MAX_CNT = 1500 -# Placement group needs to be the same for an entire bulk_insert hence -# if placement is used the actual BULK_INSERT_LIMIT will be -# max([1000, PLACEMENT_MAX_CNT]) -BULK_INSERT_LIMIT = 5000 - -# https://cloud.google.com/compute/docs/instance-groups#types_of_managed_instance_groups -ZONAL_MIG_SIZE_LIMIT = 1000 - - -@dataclass(frozen=True) -class ResumeJobData: - job_id: int - partition: str - nodes_alloc: List[str] - -@dataclass(frozen=True) -class ResumeData: - jobs: List[ResumeJobData] - - -def get_resume_file_data() -> Optional[ResumeData]: - if not (path := os.getenv("SLURM_RESUME_FILE")): - log.error("SLURM_RESUME_FILE was not in environment. Cannot get detailed job, node, partition allocation data.") - return None - blob = Path(path).read_text() - log.debug(f"Resume data: {blob}") - data = json.loads(blob) - - jobs = [] - for jo in data.get("jobs", []): - job = ResumeJobData( - job_id = jo.get("job_id"), - partition = jo.get("partition"), - nodes_alloc = util.to_hostnames(jo.get("nodes_alloc")), - ) - jobs.append(job) - return ResumeData(jobs=jobs) - -def instance_properties(nodeset: NSDict, model:str, placement_group:Optional[str], labels:Optional[dict], job_id:Optional[int]): - props = NSDict() - - if labels: # merge in extra labels on instance and disks - template_link = lookup().node_template(model) - template_info = lookup().template_info(template_link) - - props.labels = {**template_info.labels, **labels} - - for disk in template_info.disks: - if disk.initializeParams.get("diskType", "local-ssd") == "local-ssd": - continue # do not label local ssd - disk.initializeParams.labels.update(labels) - props.disks = template_info.disks - - if placement_group: - props.resourcePolicies = [placement_group] - - if reservation := lookup().nodeset_reservation(nodeset): - update_reservation_props(reservation, props, placement_group, reservation.calendar) - - if (fr := lookup().future_reservation(nodeset)) and fr.specific: - assert fr.active_reservation - update_reservation_props(fr.active_reservation, props, placement_group, fr.calendar) - - if props.resourcePolicies: - props.scheduling.onHostMaintenance = "TERMINATE" - - if nodeset.maintenance_interval: - props.scheduling.maintenanceInterval = nodeset.maintenance_interval - - if nodeset.dws_flex.enabled and nodeset.dws_flex.use_bulk_insert: - update_props_dws(props, nodeset.dws_flex, job_id) - - # Override with properties explicit specified in the nodeset - props.update(nodeset.get("instance_properties") or {}) - return props - -def update_reservation_props(reservation:ReservationDetails, props:NSDict, placement_group:Optional[str], calendar_mode:bool) -> None: - props.reservationAffinity = { - "consumeReservationType": "SPECIFIC_RESERVATION", - "key": f"compute.{util.universe_domain()}/reservation-name", - "values": [reservation.bulk_insert_name], - } - - if reservation.dense or calendar_mode: - props.scheduling.provisioningModel = "RESERVATION_BOUND" - - # Figure out `resourcePolicies` - if reservation.policies: # use ones already attached to reservations - props.resourcePolicies = reservation.policies - elif reservation.dense and placement_group: # use once created by Slurm - props.resourcePolicies = [placement_group] - else: # vanilla reservations don't support external policies - props.resourcePolicies = [] - log.info( - f"reservation {reservation.bulk_insert_name} is being used with resourcePolicies: {props.resourcePolicies}") - -def update_props_dws(props: NSDict, dws_flex: NSDict, job_id: Optional[int]) -> None: - props.scheduling.onHostMaintenance = "TERMINATE" - props.scheduling.instanceTerminationAction = "DELETE" - props.reservationAffinity['consumeReservationType'] = "NO_RESERVATION" - props.scheduling.maxRunDuration['seconds'] = dws_flex_duration(dws_flex, job_id) - -def dws_flex_duration(dws_flex: NSDict, job_id: Optional[int]) -> int: - max_duration = dws_flex.max_run_duration - if dws_flex.use_job_duration and job_id is not None and (job := lookup().job(job_id)) and job.duration: - if timedelta(seconds=30) <= job.duration <= timedelta(weeks=1): - max_duration = int(job.duration.total_seconds()) - else: - log.info("Job TimeLimit cannot be less than 30 seconds or exceed one week") - return max_duration - -def create_instances_request(nodes: List[str], placement_group: Optional[str], excl_job_id: Optional[int]): - """Call regionInstances.bulkInsert to create instances""" - assert 0 < len(nodes) <= BULK_INSERT_LIMIT - - # model here indicates any node that can be used to describe the rest - model = next(iter(nodes)) - log.debug(f"create_instances_request: {model} placement: {placement_group}") - - nodeset = lookup().node_nodeset(model) - template = lookup().node_template(model) - labels = {"slurm_job_id": excl_job_id} if excl_job_id else None - - body = dict( - count = len(nodes), - sourceInstanceTemplate = template, - # key is instance name, value overwrites properties (no overwrites) - perInstanceProperties = {k: {} for k in nodes}, - instanceProperties = instance_properties( - nodeset, model, placement_group, labels, excl_job_id - ), - ) - - if placement_group and excl_job_id is not None: - pass # do not set minCount to force "all or nothing" behavior - else: - body["minCount"] = 1 - - zone_allow = nodeset.zone_policy_allow or [] - zone_deny = nodeset.zone_policy_deny or [] - - if len(zone_allow) == 1: # if only one zone is used, use zonal BulkInsert API, as less prone to errors - api_method = lookup().compute.instances().bulkInsert - method_args = {"zone": zone_allow[0]} - else: - api_method = lookup().compute.regionInstances().bulkInsert - method_args = {"region": lookup().node_region(model)} - - body["locationPolicy"] = dict( - locations = { - **{ f"zones/{z}": {"preference": "ALLOW"} for z in zone_allow }, - **{ f"zones/{z}": {"preference": "DENY"} for z in zone_deny }}, - targetShape = nodeset.zone_target_shape, - ) - - req = api_method( - project=lookup().project, - body=body, - **method_args) - log.debug(f"new request: endpoint={req.methodId} nodes={to_hostlist(nodes)}") - log_api_request(req) - return req - -@dataclass() -class PlacementAndNodes: - placement: Optional[str] - nodes: List[str] - -@dataclass(frozen=True) -class BulkChunk: - nodes: List[str] - prefix: str # - - chunk_idx: int - excl_job_id: Optional[int] - placement_group: Optional[str] = None - - @property - def name(self): - if self.placement_group is not None: - return f"{self.prefix}:job{self.excl_job_id}:{self.placement_group}:{self.chunk_idx}" - if self.excl_job_id is not None: - return f"{self.prefix}:job{self.excl_job_id}:{self.chunk_idx}" - return f"{self.prefix}:{self.chunk_idx}" - - -def group_nodes_bulk(nodes: List[str], resume_data: Optional[ResumeData], lkp: util.Lookup): - """group nodes by nodeset, placement_group, exclusive_job_id if any""" - if resume_data is None: # all nodes will be considered jobless - resume_data = ResumeData(jobs=[]) - - nodes_set = set(nodes) # turn into set to simplify intersection - non_excl = nodes_set.copy() - groups : Dict[Optional[int], List[PlacementAndNodes]] = {} # excl_job_id|none -> PlacementAndNodes - - # expand all exclusive job nodelists - for job in resume_data.jobs: - if not lkp.cfg.partitions[job.partition].enable_job_exclusive: - continue - - groups[job.job_id] = [] - # placement group assignment is based on all allocated nodes, ... - for pn in create_placements(job.nodes_alloc, job.job_id, lkp): - groups[job.job_id].append( - PlacementAndNodes( - placement=pn.placement, - #... but we only want to handle nodes in nodes_resume in this run. - nodes = sorted(set(pn.nodes) & nodes_set) - )) - non_excl.difference_update(job.nodes_alloc) - - groups[None] = create_placements(sorted(non_excl), excl_job_id=None, lkp=lkp) - - def chunk_nodes(nodes: List[str]): - if not nodes: - return [] - - model = nodes[0] - - if lkp.is_flex_node(model): - chunk_size = ZONAL_MIG_SIZE_LIMIT - elif lkp.node_is_tpu(model): - ns_name = lkp.node_nodeset_name(model) - chunk_size = tpu.TPU.make(ns_name, lkp).vmcount - else: - chunk_size = BULK_INSERT_LIMIT - - return chunked(nodes, n=chunk_size) - - chunks = [ - BulkChunk( - nodes=nodes_chunk, - prefix=lkp.node_prefix(nodes_chunk[0]), # - - excl_job_id = job_id, - placement_group=pn.placement, - chunk_idx=i) - - for job_id, placements in groups.items() - for pn in placements if pn.nodes - for i, nodes_chunk in enumerate(chunk_nodes(pn.nodes)) - ] - return {chunk.name: chunk for chunk in chunks} - - -def resume_nodes(nodes: List[str], resume_data: Optional[ResumeData]): - """resume nodes in nodelist""" - lkp = lookup() - # Prevent dormant nodes associated with a future reservation from being resumed - nodes, dormant_fr_nodes = util.separate(lkp.is_dormant_fr_node, nodes) - - if dormant_fr_nodes: - log.warning(f"Resume was unable to resume future reservation nodes={dormant_fr_nodes}") - down_nodes_notify_jobs(dormant_fr_nodes, "Reservation is not active, nodes cannot be resumed", resume_data) - - nodes, flex_managed = util.separate(lkp.is_provisioning_flex_node, nodes) - if flex_managed: - log.warning(f"Resume was unable to resume nodes={flex_managed} already managed by MIGs") - down_nodes_notify_jobs(flex_managed, "VM is managed MIG, can not be resumed", resume_data) - - if not nodes: - log.info("No nodes to resume") - return - - nodes = sorted(nodes, key=lkp.node_prefix) - grouped_nodes = group_nodes_bulk(nodes, resume_data, lkp) - - if log.isEnabledFor(logging.DEBUG): - grouped_nodelists = { - group: to_hostlist(chunk.nodes) for group, chunk in grouped_nodes.items() - } - log.debug( - "node bulk groups: \n{}".format(yaml.safe_dump(grouped_nodelists).rstrip()) - ) - - tpu_chunks, flex_chunks = [], [] - bi_inserts = {} - - for group, chunk in grouped_nodes.items(): - model = chunk.nodes[0] - - if lkp.node_is_tpu(model): - tpu_chunks.append(chunk.nodes) - elif lkp.is_flex_node(model): - flex_chunks.append(chunk) - else: - bi_inserts[group] = create_instances_request( - chunk.nodes, chunk.placement_group, chunk.excl_job_id - ) - - for chunk in flex_chunks: - mig_flex.resume_flex_chunk(chunk.nodes, chunk.excl_job_id, lkp) - - # execute all bulkInsert requests with batch - bulk_ops = dict( - zip(bi_inserts.keys(), map_with_futures(ensure_execute, bi_inserts.values())) - ) - log.debug(f"bulk_ops={yaml.safe_dump(bulk_ops)}") - started = { - group: op for group, op in bulk_ops.items() if not isinstance(op, Exception) - } - failed = { - group: err for group, err in bulk_ops.items() if isinstance(err, Exception) - } - if failed: - failed_reqs = [str(e) for e in failed.items()] - log.error("bulkInsert API failures: {}".format("; ".join(failed_reqs))) - for ident, exc in failed.items(): - down_nodes_notify_jobs(grouped_nodes[ident].nodes, f"GCP Error: {exc._get_reason()}", resume_data) # type: ignore - - if log.isEnabledFor(logging.DEBUG): - for group, op in started.items(): - group_nodes = grouped_nodelists[group] - name = op["name"] - gid = op["operationGroupId"] - log.debug( - f"new bulkInsert operation started: group={group} nodes={group_nodes} name={name} operationGroupId={gid}" - ) - # wait for all bulkInserts to complete and log any errors - bulk_operations = {group: wait_for_operation(op) for group, op in started.items()} - - # Start TPU after regular nodes so that regular nodes are not affected by the slower TPU nodes - execute_with_futures(tpu.start_tpu, tpu_chunks) - - for group, op in bulk_operations.items(): - _handle_bulk_insert_op(op, grouped_nodes[group].nodes, resume_data) - - -def _get_failed_zonal_instance_inserts(bulk_op: Any, zone: str, lkp: util.Lookup) -> list[Any]: - group_id = bulk_op["operationGroupId"] - user = bulk_op["user"] - started = bulk_op["startTime"] - ended = bulk_op["endTime"] - - fltr = f'(user eq "{user}") AND (operationType eq "insert") AND (creationTimestamp > "{started}") AND (creationTimestamp < "{ended}")' - act = lkp.compute.zoneOperations() - req = act.list(project=lkp.project, zone=zone, filter=fltr) - ops = [] - while req is not None: - result = util.ensure_execute(req) - for op in result.get("items", []): - if op.get("operationGroupId") == group_id and "error" in op: - ops.append(op) - req = act.list_next(req, result) - return ops - - -def _get_failed_instance_inserts(bulk_op: Any, lkp: util.Lookup) -> list[Any]: - zones = set() # gather zones that had failed inserts - for loc, stat in bulk_op.get("instancesBulkInsertOperationMetadata", {}).get("perLocationStatus", {}).items(): - pref, zone = loc.split("/", 1) - if not pref == "zones": - log.error(f"Unexpected location: {loc} in operation {bulk_op['name']}") - continue - if stat.get("targetVmCount", 0) != stat.get("createdVmCount", 0): - zones.add(zone) - - res = [] - for zone in zones: - res.extend(_get_failed_zonal_instance_inserts(bulk_op, zone, lkp)) - return res - -def _handle_bulk_insert_op(op: Dict, nodes: List[str], resume_data: Optional[ResumeData]) -> None: - """ - Handles **DONE** BulkInsert operations - """ - assert op["operationType"] == "bulkInsert" and op["status"] == "DONE", f"unexpected op: {op}" - - group_id = op["operationGroupId"] - if "error" in op: - error = op["error"]["errors"][0] - log.error( - f"bulkInsert operation error: {error['code']} name={op['name']} operationGroupId={group_id} nodes={to_hostlist(nodes)}" - ) - - created = 0 - for status in op["instancesBulkInsertOperationMetadata"]["perLocationStatus"].values(): - created += status.get("createdVmCount", 0) - if created == len(nodes): - log.info(f"created {len(nodes)} instances: nodes={to_hostlist(nodes)}") - return # no need to gather status of insert-operations. - - # TODO: don't gather insert-operations per bulkInsert request, instead aggregate it - # across all bulkInserts (goes one level above this function) - failed = _get_failed_instance_inserts(op, util.lookup()) - - # Multiple errors are possible, group by all of them (joined string codes) - by_error_inserts = util.groupby_unsorted( - failed, - lambda op: "+".join(err["code"] for err in op["error"]["errors"]), - ) - for code, failed_ops in by_error_inserts: - failed_ops = list(failed_ops) - failed_nodes = [trim_self_link(op["targetLink"]) for op in failed_ops] - hostlist = util.to_hostlist(failed_nodes) - log.error( - f"{len(failed_nodes)} instances failed to start: {code} ({hostlist}) operationGroupId={group_id}" - ) - - msg = "; ".join( - f"{err['code']}: {err['message'] if 'message' in err else 'no message'}" - for err in failed_ops[0]["error"]["errors"] - ) - if code != "RESOURCE_ALREADY_EXISTS": - down_nodes_notify_jobs(failed_nodes, f"GCP Error: {msg}", resume_data) - log.error( - f"errors from insert for node '{failed_nodes[0]}' ({failed_ops[0]['name']}): {msg}" - ) - - -def down_nodes_notify_jobs(nodes: List[str], reason: str, resume_data: Optional[ResumeData]) -> None: - """set nodes down with reason""" - nodes_set = set(nodes) # turn into set to speed up intersection - jobs = resume_data.jobs if resume_data else [] - reason_quoted = shlex.quote(reason) - - for job in jobs: - if not (set(job.nodes_alloc) & nodes_set): - continue - run(f"{lookup().scontrol} update jobid={job.job_id} admincomment={reason_quoted}", check=False) - run(f"{lookup().scontrol} notify {job.job_id} {reason_quoted}", check=False) - - nodelist = util.to_hostlist(nodes) - log.error(f"Marking nodes {nodelist} as DOWN, reason: {reason}") - run(f"{lookup().scontrol} update nodename={nodelist} state=down reason={reason_quoted}", check=False) - - - - -def create_placement_request(pg_name: str, region: str, max_distance: Optional[int]): - config = { - "name": pg_name, - "region": region, - "groupPlacementPolicy": { - "collocation": "COLLOCATED", - "maxDistance": max_distance - }, - } - - request = lookup().compute.resourcePolicies().insert( - project=lookup().project, region=region, body=config - ) - log_api_request(request) - return request - - -def create_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: - nodeset_map = collections.defaultdict(list) - for node in nodes: # split nodes on nodesets - nodeset_map[lkp.node_nodeset_name(node)].append(node) - - placements = [] - for _, ns_nodes in nodeset_map.items(): - placements.extend(create_nodeset_placements(ns_nodes, excl_job_id, lkp)) - return placements - - -def _allocate_nodes_to_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: - # canned result for no placement policies created - no_pp = [PlacementAndNodes(placement=None, nodes=nodes)] - - if excl_job_id and len(nodes) < 2: - return no_pp # don't create placement_policy for just one node - - model = nodes[0] - nodeset = lkp.node_nodeset(model) - - if lkp.is_flex_node(model): - return no_pp # TODO(FLEX): Add support for workload policies - if lkp.node_is_tpu(model): - return no_pp - if not (nodeset.enable_placement and valid_placement_node(model)): - return no_pp - - max_count = calculate_chunk_size(nodeset, lkp) - - name_prefix = f"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-{nodeset.nodeset_name}" - if excl_job_id: # simply chunk given nodes by max size of placement - return [ - PlacementAndNodes(placement=f"{name_prefix}-{excl_job_id}-{i}", nodes=chunk) - for i, chunk in enumerate(chunked(nodes, n=max_count)) - ] - - # split whole nodeset (not only nodes to resume) into chunks of max size of placement - # create placements (most likely already exists) placements for requested nodes - chunks = collections.defaultdict(list) # chunk_id -> nodes - invalid = [] - - for node in nodes: - try: - chunk = lkp.node_index(node) // max_count - chunks[chunk].append(node) - except: - invalid.append(node) - - placements = [ - # NOTE: use 0 instead of job_id for consistency with previous SlurmGCP behavior - PlacementAndNodes(placement=f"{name_prefix}-0-{c_id}", nodes=c_nodes) - for c_id, c_nodes in chunks.items() - ] - - if invalid: - placements.append(PlacementAndNodes(placement=None, nodes=invalid)) - log.error(f"Could not find placement for nodes with unexpected names: {to_hostlist(invalid)}") - - return placements - -def calculate_chunk_size(nodeset: NSDict, lkp: util.Lookup) -> int: - # Calculates the chunk size based on max distance value received - machine_type = lkp.template_info(nodeset.instance_template).machine_type.family - max_distance = nodeset.placement_max_distance - if max_distance == 1: - return 22 - elif max_distance == 2: - if machine_type.startswith("a3"): - return 256 - else: - return 150 - elif max_distance == 3: - return 1500 - else: - return PLACEMENT_MAX_CNT - -def create_nodeset_placements(nodes: List[str], excl_job_id:Optional[int], lkp: util.Lookup) -> List[PlacementAndNodes]: - placements = _allocate_nodes_to_placements(nodes, excl_job_id, lkp) - region = lkp.node_region(nodes[0]) - max_distance = lkp.node_nodeset(nodes[0]).get('placement_max_distance') - - if log.isEnabledFor(logging.DEBUG): - debug_p = {p.placement: to_hostlist(p.nodes) for p in placements} - log.debug( - f"creating {len(placements)} placement groups: \n{yaml.safe_dump(debug_p).rstrip()}" - ) - - requests = { - p.placement: create_placement_request(p.placement, region, max_distance) for p in placements if p.placement - } - if not requests: - return placements - # TODO: aggregate all requests for whole resume and execute them at once (don't limit to nodeset/job) - ops = dict( - zip(requests.keys(), map_with_futures(ensure_execute, requests.values())) - ) - - def classify_result(item): - op = item[1] - if not isinstance(op, Exception): - return "submitted" - if all(e.get("reason") == "alreadyExists" for e in op.error_details): # type: ignore - return "redundant" - return "failed" - - grouped_ops = dict(util.groupby_unsorted(list(ops.items()), classify_result)) - submitted, redundant, failed = ( - dict(grouped_ops.get(key, {})) for key in ("submitted", "redundant", "failed") - ) - if redundant: - log.warning( - "placement policies already exist: {}".format(",".join(redundant.keys())) - ) - if failed: - reqs = [f"{e}" for _, e in failed.values()] - log.fatal("failed to create placement policies: {}".format("; ".join(reqs))) - operations = {group: wait_for_operation(op) for group, op in submitted.items()} - for group, op in operations.items(): - if "error" in op: - msg = "; ".join( - f"{err['code']}: {err['message'] if 'message' in err else 'no message'}" - for err in op["error"]["errors"] - ) - log.error( - f"placement group failed to create: '{group}' ({op['name']}): {msg}" - ) - - log.info( - f"created {len(operations)} placement groups ({to_hostlist(operations.keys())})" - ) - return placements - - -def valid_placement_node(node: str) -> bool: - invalid_types = frozenset(["e2", "t2d", "n1", "t2a", "m1", "m2", "m3"]) - mt = lookup().node_template_info(node).machineType - if mt.split("-")[0] in invalid_types: - log.warn(f"Unsupported machine type for placement policy: {mt}.") - log.warn( - f"Please do not use any the following machine types with placement policy: ({','.join(invalid_types)})" - ) - return False - return True - - -def main(nodelist: str) -> None: - """main called when run as script""" - log.debug(f"ResumeProgram {nodelist}") - # Filter out nodes not in config.yaml - other_nodes, nodes = separate( - lookup().is_power_managed_node, util.to_hostnames(nodelist) - ) - if other_nodes: - log.error( - f"Ignoring non-power-managed nodes '{to_hostlist(other_nodes)}' from '{nodelist}'" - ) - - if not nodes: - log.info("No nodes to resume") - return - resume_data = get_resume_file_data() - log.info(f"resume {util.to_hostlist(nodes)}") - resume_nodes(nodes, resume_data) - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("nodelist", help="list of nodes to resume") - args = util.init_log_and_parse(parser) - main(args.nodelist) diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh deleted file mode 100644 index a180761914..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/resume_wrapper.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) -PYTHON_SCRIPT="${SCRIPT_DIR}/resume.py" - -# Capture all arguments passed by Slurm (the nodelist). -ALL_ARGS=("$@") - -# This array will hold extra argument for resume.py, like the resume data file. -UNIQUE_RESUME_FILE="" - -# Handle SLURM_RESUME_FILE if provided -if [ -n "${SLURM_RESUME_FILE-}" ] && [ -f "$SLURM_RESUME_FILE" ]; then - SAFE_DIR="/tmp/slurm_resume_data" - mkdir -p "$SAFE_DIR" - - UNIQUE_RESUME_FILE="${SAFE_DIR}/resumedata.$$.json" - cp "$SLURM_RESUME_FILE" "$UNIQUE_RESUME_FILE" -fi - -SLURM_RESUME_FILE="${UNIQUE_RESUME_FILE}" -"${PYTHON_SCRIPT}" "${ALL_ARGS[@]}" & -disown - -exit 0 diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py deleted file mode 100644 index 0d27e6cfac..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup.py +++ /dev/null @@ -1,648 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import logging -import os -import shutil -import subprocess -import stat -import time -import yaml -from pathlib import Path -import functools - -import util -from util import ( - lookup, - dirs, - slurmdirs, - run, - install_custom_scripts, -) -import conf -import slurmsync - -from setup_network_storage import ( - setup_network_storage, - setup_nfs_exports, -) - - -log = logging.getLogger() - - -MOTD_HEADER = """ - SSSSSSS - SSSSSSSSS - SSSSSSSSS - SSSSSSSSS - SSSS SSSSSSS SSSS - SSSSSS SSSSSS - SSSSSS SSSSSSS SSSSSS - SSSS SSSSSSSSS SSSS - SSS SSSSSSSSS SSS - SSSSS SSSS SSSSSSSSS SSSS SSSSS - SSS SSSSSS SSSSSSSSS SSSSSS SSS - SSSSSS SSSSSSS SSSSSS - SSS SSSSSS SSSSSS SSS - SSSSS SSSS SSSSSSS SSSS SSSSS - S SSS SSSSSSSSS SSS S - SSS SSSS SSSSSSSSS SSSS SSS - S SSS SSSSSS SSSSSSSSS SSSSSS SSS S - SSSSS SSSSSS SSSSSSSSS SSSSSS SSSSS - S SSSSS SSSS SSSSSSS SSSS SSSSS S - S SSS SSS SSS SSS S - S S S S - SSS - SSS - SSS - SSS - SSSSSSSSSSSS SSS SSSS SSSS SSSSSSSSS SSSSSSSSSSSSSSSSSSSS -SSSSSSSSSSSSS SSS SSSS SSSS SSSSSSSSSS SSSSSSSSSSSSSSSSSSSSSS -SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS -SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS -SSSSSSSSSSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS - SSSSSSSSSSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS - SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS - SSSS SSS SSSS SSSS SSSS SSSS SSSS SSSS -SSSSSSSSSSSSS SSS SSSSSSSSSSSSSSS SSSS SSSS SSSS SSSS -SSSSSSSSSSSS SSS SSSSSSSSSSSSS SSSS SSSS SSSS SSSS - -""" -_MAINTENANCE_SBATCH_SCRIPT_PATH = dirs.custom_scripts / "perform_maintenance.sh" - -def start_motd(): - """advise in motd that slurm is currently configuring""" - wall_msg = "*** Slurm is currently being configured in the background. ***" - motd_msg = MOTD_HEADER + wall_msg + "\n\n" - Path("/etc/motd").write_text(motd_msg) - util.run(f"wall -n '{wall_msg}'", timeout=30) - - -def end_motd(broadcast=True): - """modify motd to signal that setup is complete""" - Path("/etc/motd").write_text(MOTD_HEADER) - - if not broadcast: - return - - run( - "wall -n '*** Slurm {} setup complete ***'".format(lookup().instance_role), - timeout=30, - ) - if not lookup().is_controller: - run( - """wall -n ' -/home on the controller was mounted over the existing /home. -Log back in to ensure your home directory is correct. -'""", - timeout=30, - ) - - -def failed_motd(): - """modify motd to signal that setup is failed""" - wall_msg = f"*** Slurm setup failed! Please view log: {util.get_log_path()} ***" - motd_msg = MOTD_HEADER + wall_msg + "\n\n" - Path("/etc/motd").write_text(motd_msg) - util.run(f"wall -n '{wall_msg}'", timeout=30) - - -def _startup_script_timeout(lkp: util.Lookup) -> int: - if lkp.is_controller: - return lkp.cfg.get("controller_startup_scripts_timeout", 300) - elif lkp.instance_role == "compute": - return lkp.cfg.get("compute_startup_scripts_timeout", 300) - elif lkp.is_login_node: - return lkp.cfg.login_groups[util.instance_login_group()].get("startup_scripts_timeout", 300) - return 300 - - -def run_custom_scripts(): - """run custom scripts based on instance_role""" - custom_dir = dirs.custom_scripts - if lookup().is_controller: - # controller has all scripts, but only runs controller.d - custom_dirs = [custom_dir / "controller.d"] - elif lookup().instance_role == "compute": - # compute setup with nodeset.d - custom_dirs = [custom_dir / "nodeset.d"] - elif lookup().is_login_node: - # login setup with only login.d - custom_dirs = [custom_dir / "login.d"] - else: - # Unknown role: run nothing - custom_dirs = [] - - timeout = _startup_script_timeout(lookup()) - - custom_scripts = [ - p - for d in custom_dirs - for p in d.rglob("*") - if p.is_file() and not p.name.endswith(".disabled") - ] - print_scripts = ",".join(str(s.relative_to(custom_dir)) for s in custom_scripts) - log.debug(f"custom scripts to run: {custom_dir}/({print_scripts})") - - try: - for script in custom_scripts: - log.info(f"running script {script.name} with timeout={timeout}") - result = run(str(script), timeout=timeout, check=False, shell=True) - runlog = ( - f"{script.name} returncode={result.returncode}\n" - f"stdout={result.stdout}stderr={result.stderr}" - ) - log.info(runlog) - result.check_returncode() - except OSError as e: - log.error(f"script {script} is not executable") - raise e - except subprocess.TimeoutExpired as e: - log.error(f"script {script} did not complete within timeout={timeout}") - raise e - except Exception as e: - log.exception(f"script {script} encountered an exception") - raise e - -def mount_save_state_disk(): - disk_name = f"/dev/disk/by-id/google-{lookup().cfg.controller_state_disk.device_name}" - mount_point = util.slurmdirs.state - fs_type = "ext4" - - rdevice = util.run(f"realpath {disk_name}").stdout.strip() - file_output = util.run(f"file -s {rdevice}").stdout.strip() - if "filesystem" not in file_output: - util.run(f"mkfs -t {fs_type} -q {rdevice}") - - fstab_entry = f"{disk_name} {mount_point} {fs_type}" - with open("/etc/fstab", "r") as f: - fstab = f.readlines() - if fstab_entry not in fstab: - with open("/etc/fstab", "a") as f: - f.write(f"{fstab_entry} defaults 0 0\n") - - util.run(f"systemctl daemon-reload") - - os.makedirs(mount_point, exist_ok=True) - util.run(f"mount {mount_point}") - - util.chown_slurm(mount_point) - - -def setup_jwt_key(): - jwt_key = Path(slurmdirs.state / "jwt_hs256.key") - - if jwt_key.exists(): - log.info("JWT key already exists. Skipping key generation.") - else: - run("dd if=/dev/urandom bs=32 count=1 > " + str(jwt_key), shell=True) - - util.chown_slurm(jwt_key, mode=0o400) - - -def _generate_key(p: Path) -> None: - run(f"dd if=/dev/random of={p} bs=1024 count=1") - - -def setup_key(lkp: util.Lookup) -> None: - file_name = "munge.key" - dir = dirs.munge - - if lkp.cfg.enable_slurm_auth: - file_name = "slurm.key" - dir = slurmdirs.etc - - dst = Path(dir / file_name) - - if lkp.cfg.controller_state_disk.device_name: - # Copy key from persistent state disk - persist = slurmdirs.state / file_name - if not persist.exists(): - _generate_key(persist) - - shutil.copyfile(persist, dst) - if lkp.cfg.enable_slurm_auth: - util.chown_slurm(dst, mode=0o400) - util.chown_slurm(persist, mode=0o400) - else: - shutil.chown(dst, user="munge", group="munge") - os.chmod(dst, stat.S_IRUSR) - else: - if dst.exists(): - log.info("key already exists. Skipping key generation.") - else: - _generate_key(dst) - if lkp.cfg.enable_slurm_auth: - util.chown_slurm(dst, mode=0o400) - else: - shutil.chown(dst, user="munge", group="munge") - os.chmod(dst, stat.S_IRUSR) - - if lkp.cfg.enable_slurm_auth: - # Put key into shared volume for distribution - distributed = util.slurmdirs.key_distribution / file_name - shutil.copyfile(dst, distributed) - util.chown_slurm(distributed, mode=0o400) - # Munge is distributed from /etc/munge. - else: - run("systemctl restart munge", timeout=30) - - -def setup_nss_slurm(): - """install and configure nss_slurm""" - # setup nss_slurm - util.mkdirp(Path("/var/spool/slurmd")) - run( - "ln -s {}/lib/libnss_slurm.so.2 /usr/lib64/libnss_slurm.so.2".format( - slurmdirs.prefix - ), - check=False, - ) - run(r"sed -i 's/\(^\(passwd\|group\):\s\+\)/\1slurm /g' /etc/nsswitch.conf") - - -def setup_sudoers(): - content = """ -# Allow SlurmUser to manage the slurm daemons -slurm ALL= NOPASSWD: /usr/bin/systemctl restart slurmd.service -slurm ALL= NOPASSWD: /usr/bin/systemctl restart sackd.service -slurm ALL= NOPASSWD: /usr/bin/systemctl restart slurmctld.service -""" - sudoers_file = Path("/etc/sudoers.d/slurm") - sudoers_file.write_text(content) - sudoers_file.chmod(0o0440) - - -def setup_maintenance_script(): - perform_maintenance = """#!/bin/bash - -#SBATCH --priority=low -#SBATCH --time=180 - -VM_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google") -ZONE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor: Google" | cut -d '/' -f 4) - -gcloud compute instances perform-maintenance $VM_NAME \ - --zone=$ZONE -""" - - - with open(_MAINTENANCE_SBATCH_SCRIPT_PATH, "w") as f: - f.write(perform_maintenance) - - util.chown_slurm(_MAINTENANCE_SBATCH_SCRIPT_PATH, mode=0o755) - - -def update_system_config(file, content): - """Add system defaults options for service files""" - sysconfig = Path("/etc/sysconfig") - default = Path("/etc/default") - - if sysconfig.exists(): - conf_dir = sysconfig - elif default.exists(): - conf_dir = default - else: - raise Exception("Cannot determine system configuration directory.") - - slurmd_file = Path(conf_dir, file) - slurmd_file.write_text(content) - - -def configure_mysql(): - cnfdir = Path("/etc/my.cnf.d") - if not cnfdir.exists(): - cnfdir = Path("/etc/mysql/conf.d") - if not (cnfdir / "mysql_slurm.cnf").exists(): - (cnfdir / "mysql_slurm.cnf").write_text( - """ -[mysqld] -bind-address=127.0.0.1 -innodb_buffer_pool_size=1024M -innodb_log_file_size=64M -innodb_lock_wait_timeout=900 -""" - ) - run("systemctl enable mariadb", timeout=30) - run("systemctl restart mariadb", timeout=30) - - mysql, host = "mysql -u root -e", lookup().control_host - run(f"""{mysql} "drop user if exists 'slurm'@'localhost'";""", timeout=30) - run(f"""{mysql} "create user 'slurm'@'localhost'";""", timeout=30) - run(f"""{mysql} "grant all on slurm_acct_db.* TO 'slurm'@'localhost'";""", timeout=30) - run(f"""{mysql} "drop user if exists 'slurm'@'{host}'";""", timeout=30) - run(f"""{mysql} "create user 'slurm'@'{host}'";""", timeout=30) - run(f"""{mysql} "grant all on slurm_acct_db.* TO 'slurm'@'{host}'";""", timeout=30) - - -def configure_dirs(): - for p in dirs.values(): - util.mkdirp(p) - - for p in (dirs.slurm, dirs.scripts, dirs.custom_scripts): - util.chown_slurm(p) - - for p in slurmdirs.values(): - util.mkdirp(p) - util.chown_slurm(p) - - for sl, tgt in ( # create symlinks - (Path("/etc/slurm"), slurmdirs.etc), - (dirs.scripts / "etc", slurmdirs.etc), - (dirs.scripts / "log", dirs.log), - ): - if sl.exists() and sl.is_symlink(): - sl.unlink() - sl.symlink_to(tgt) - - # copy auxiliary scripts - for dst_folder, src_file in ((lookup().cfg.slurm_bin_dir, - Path("sort_nodes.py")), - (dirs.custom_scripts / "task_prolog.d", - Path("tools/task-prolog")), - (dirs.custom_scripts / "task_epilog.d", - Path("tools/task-epilog"))): - dst = Path(dst_folder) / src_file.name - util.mkdirp(dst.parent) - shutil.copyfile(util.scripts_dir / src_file, dst) - os.chmod(dst, 0o755) - - -def self_report_controller_address(lkp: util.Lookup) -> None: - if not lkp.cfg.controller_network_attachment: - return # only self report address if network attachment is used - data = { "slurm_control_addr": lkp.cfg.slurm_control_addr } - bucket, prefix = util._get_bucket_and_common_prefix() - blob = util.storage_client().bucket(bucket).blob(f"{prefix}/controller_addr.yaml") - with blob.open('w') as f: - f.write(yaml.dump(data)) - -def setup_controller(): - """Run controller setup""" - log.info("Setting up controller") - lkp = util.lookup() - util.chown_slurm(dirs.scripts / "config.yaml", mode=0o600) - install_custom_scripts() - conf.gen_controller_configs(lkp) - - if lkp.cfg.controller_state_disk.device_name != None: - mount_save_state_disk() - - setup_jwt_key() - setup_key(lkp) - - setup_sudoers() - setup_network_storage() - - run_custom_scripts() - - if not lkp.cfg.cloudsql_secret: - configure_mysql() - - run("systemctl enable slurmdbd", timeout=30) - run("systemctl restart slurmdbd", timeout=30) - - # Wait for slurmdbd to come up - time.sleep(5) - - sacctmgr = f"{slurmdirs.prefix}/bin/sacctmgr -i" - result = run( - f"{sacctmgr} add cluster {lkp.cfg.slurm_cluster_name}", timeout=30, check=False - ) - if "already exists" in result.stdout: - log.info(result.stdout) - elif result.returncode > 1: - result.check_returncode() # will raise error - - run("systemctl enable slurmctld", timeout=30) - run("systemctl restart slurmctld", timeout=30) - - run("systemctl enable slurmrestd", timeout=30) - run("systemctl restart slurmrestd", timeout=30) - - # Export at the end to signal that everything is up - run("systemctl enable nfs-server", timeout=30) - run("systemctl start nfs-server", timeout=30) - - setup_nfs_exports() - run("systemctl enable --now slurmcmd.timer", timeout=30) - - log.info("Check status of cluster services") - if not lkp.cfg.enable_slurm_auth: - run("systemctl status munge", timeout=30) - run("systemctl status slurmdbd", timeout=30) - run("systemctl status slurmctld", timeout=30) - run("systemctl status slurmrestd", timeout=30) - - try: - slurmsync.sync_instances() - except Exception: - log.exception("Failed to sync instances, will try next time.") - - run("systemctl enable slurm_load_bq.timer", timeout=30) - run("systemctl start slurm_load_bq.timer", timeout=30) - run("systemctl status slurm_load_bq.timer", timeout=30) - - # Add script to perform maintenance - setup_maintenance_script() - - self_report_controller_address(lkp) - - log.info("Done setting up controller") - pass - - -def setup_login(): - """run login node setup""" - log.info("Setting up login") - - lkp = lookup() - slurmctld_host = f"{lkp.control_host}" - if lkp.control_addr: - slurmctld_host = f"{lkp.control_host}({lkp.control_addr})" - sackd_options = [ - f'--conf-server="{slurmctld_host}:{lkp.control_host_port}"', - ] - sysconf = f"""SACKD_OPTIONS='{" ".join(sackd_options)}'""" - update_system_config("sackd", sysconf) - install_custom_scripts() - - setup_network_storage() - setup_sudoers() - if not lkp.cfg.enable_slurm_auth: - run("systemctl restart munge", timeout=30) - run("systemctl enable sackd", timeout=30) - run("systemctl restart sackd", timeout=30) - run("systemctl enable --now slurmcmd.timer", timeout=30) - - run_custom_scripts() - - log.info("Check status of cluster services") - if not lkp.cfg.enable_slurm_auth: - run("systemctl status munge", timeout=30) - run("systemctl status sackd", timeout=30) - - log.info("Done setting up login") - - -def setup_compute(): - """run compute node setup""" - log.info("Setting up compute") - - lkp = lookup() - util.chown_slurm(dirs.scripts / "config.yaml", mode=0o600) - slurmctld_host = f"{lkp.control_host}" - if lkp.control_addr: - slurmctld_host = f"{lkp.control_host}({lkp.control_addr})" - slurmd_options = [ - f'--conf-server="{slurmctld_host}:{lkp.control_host_port}"', - ] - - try: - slurmd_feature = util.instance_metadata("attributes/slurmd_feature", silent=True) - except util.MetadataNotFoundError: - slurmd_feature = None - - if slurmd_feature is not None: - slurmd_options.append(f'--conf="Feature={slurmd_feature}"') - slurmd_options.append("-Z") - - sysconf = f"""SLURMD_OPTIONS='{" ".join(slurmd_options)}'""" - update_system_config("slurmd", sysconf) - install_custom_scripts() - - setup_nss_slurm() - setup_network_storage() - - has_gpu = run("lspci | grep --ignore-case 'NVIDIA' | wc -l", shell=True).returncode - if has_gpu: - run("nvidia-smi") - - run_custom_scripts() - - setup_sudoers() - if not lkp.cfg.enable_slurm_auth: - run("systemctl restart munge", timeout=30) - run("systemctl enable slurmd", timeout=30) - run("systemctl restart slurmd", timeout=30) - run("systemctl enable --now slurmcmd.timer", timeout=30) - - log.info("Check status of cluster services") - if not lkp.cfg.enable_slurm_auth: - run("systemctl status munge", timeout=30) - run("systemctl status slurmd", timeout=30) - - log.info("Done setting up compute") - -def setup_cloud_ops() -> None: - """Add health checks, deployment info, and updated setup path to cloud ops config.""" - cloudOpsStatus = run( - "systemctl is-active --quiet google-cloud-ops-agent.service", check=False - ).returncode - - if cloudOpsStatus != 0: - return - - with open("/etc/google-cloud-ops-agent/config.yaml", "r") as f: - file = yaml.safe_load(f) - - # Update setup receiver path - file["logging"]["receivers"]["setup"]["include_paths"] = ["/var/log/slurm/setup.log"] - - # Add chs_health_check receiver - file["logging"]["receivers"]["chs_health_check"] = { - "type": "files", - "include_paths": ["/var/log/slurm/chs_health_check.log"], - "record_log_file_path": True, - } - - cluster_info = { - 'type':'modify_fields', - 'fields': { - 'labels."cluster_name"':{ - 'static_value':f"{lookup().cfg.slurm_cluster_name}" - }, - 'labels."hostname"':{ - 'static_value': f"{lookup().hostname}" - } - } - } - - file["logging"]["processors"]["add_cluster_info"] = cluster_info - file["logging"]["service"]["pipelines"]["slurmlog_pipeline"]["processors"].append("add_cluster_info") - file["logging"]["service"]["pipelines"]["slurmlog2_pipeline"]["processors"].append("add_cluster_info") - - # Add chs_health_check to slurmlog2_pipeline - file["logging"]["service"]["pipelines"]["slurmlog2_pipeline"]["receivers"].append( - "chs_health_check" - ) - - with open("/etc/google-cloud-ops-agent/config.yaml", "w") as f: - yaml.safe_dump(file, f, sort_keys=False) - - try: - run("systemctl restart google-cloud-ops-agent.service", timeout=30) - except subprocess.TimeoutExpired: - log.error("google-cloud-ops-agent.service did not restart within 30s.") - result=run("journalctl -u google-cloud-ops-agent-opentelemetry-collector.service", timeout=30, shell=True) #Gives more in-depth logs than cloud ops agent - if result.stdout: - log.error(f"System logs for google-cloud-ops-agent-opentelemetry-collector.service:\n{result.stdout}") - raise - - -def main(): - start_motd() - - log.info("Starting setup, fetching config") - sleep_seconds = 5 - while True: - try: - _, cfg = util.fetch_config() - util.update_config(cfg) - break - except util.DeffetiveStoredConfigError as e: - log.warning(f"config is not ready yet: {e}, sleeping for {sleep_seconds}s") - except Exception as e: - log.exception(f"unexpected error while fetching config, sleeping for {sleep_seconds}s") - time.sleep(sleep_seconds) - log.info("Config fetched") - setup_cloud_ops() - configure_dirs() - # call the setup function for the instance type - { - "controller": setup_controller, - "compute": setup_compute, - "login": setup_login, - }.get( - lookup().instance_role, - lambda: log.fatal(f"Unknown node role: {lookup().instance_role}"))() - - end_motd() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--slurmd-feature", dest="slurmd_feature", help="Unused, to be removed.") - _ = util.init_log_and_parse(parser) - - try: - main() - except Exception: - log.exception("Aborting setup...") - failed_motd() diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py deleted file mode 100644 index e2561dffa1..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/setup_network_storage.py +++ /dev/null @@ -1,327 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List - -import os -import sys -import stat -import time -import logging -import uuid - -import shutil -from pathlib import Path -from concurrent.futures import as_completed -from addict import Dict as NSDict # type: ignore - -import util -from util import NSMount, lookup, run, dirs, separate -from more_executors import Executors, ExceptionRetryPolicy - - -log = logging.getLogger() - -def mounts_by_local(mounts: list[NSMount]) -> dict[str, NSMount]: - """convert list of mounts to dict of mounts, local_mount as key""" - return {str(m.local_mount.resolve()): m for m in mounts} - - -def _get_default_mounts(lkp: util.Lookup) -> list[NSMount]: - if lkp.cfg.disable_default_mounts: - return [] - return [ - NSMount( - server_ip=lkp.controller_mount_server_ip(), - remote_mount=path, - local_mount=path, - fs_type="nfs", - mount_options="defaults,hard,intr", - ) - for path in ( - dirs.home, - dirs.apps, - ) - ] - -def get_slurm_bucket_mount() -> NSMount: - bucket, path = util._get_bucket_and_common_prefix() - return NSMount( - fs_type="gcsfuse", - server_ip="", - remote_mount=Path(bucket), - local_mount=dirs.slurm_bucket_mount, - mount_options=f"defaults,_netdev,implicit_dirs,only_dir={path}", - ) - -def resolve_network_storage() -> List[NSMount]: - """Combine appropriate network_storage fields to a single list""" - lkp = lookup() - - # create dict of mounts, local_mount: mount_info - mounts = mounts_by_local(_get_default_mounts(lkp)) - - if lkp.is_controller and util.should_mount_slurm_bucket(): - mounts.update(mounts_by_local([get_slurm_bucket_mount()])) - - # On non-controller instances, entries in network_storage could overwrite - # default exports from the controller. Be careful, of course - common = [lkp.normalize_ns_mount(m) for m in lkp.cfg.network_storage] - mounts.update(mounts_by_local(common)) - - if lkp.is_login_node: - login_group = lkp.cfg.login_groups[util.instance_login_group()] - login_ns = [lkp.normalize_ns_mount(m) for m in login_group.network_storage] - mounts.update(mounts_by_local(login_ns)) - - if lkp.instance_role == "compute": - try: - nodeset = lkp.node_nodeset() - except Exception: - pass # external nodename, skip lookup - else: - nodeset_ns = [lkp.normalize_ns_mount(m) for m in nodeset.network_storage] - mounts.update(mounts_by_local(nodeset_ns)) - - return list(mounts.values()) - - -def is_controller_mount(mount) -> bool: - # NOTE: Valid Lustre server_ip can take the form of '@tcp' - server_ip = mount.server_ip.split("@")[0] - mount_addr = util.host_lookup(server_ip) - return mount_addr == lookup().control_host_addr - -def setup_network_storage(): - """prepare network fs mounts and add them to fstab""" - log.info("Set up network storage") - - all_mounts = resolve_network_storage() - if lookup().is_controller: - mounts, _ = separate(is_controller_mount, all_mounts) - else: - mounts = all_mounts - - # Determine fstab entries and write them out - fstab_entries = [] - for mount in mounts: - local_mount = mount.local_mount - fs_type = mount.fs_type - server_ip = mount.server_ip or "" - src = mount.remote_mount if fs_type == "gcsfuse" else f"{server_ip}:{mount.remote_mount}" - - log.info(f"Setting up mount ({fs_type}) {src} to {local_mount}") - util.mkdirp(local_mount) - - mount_options = mount.mount_options.split(",") if mount.mount_options else [] - if "_netdev" not in mount_options: - mount_options += ["_netdev"] - options_line = ",".join(mount_options) - - - fstab_entries.append(f"{src} {local_mount} {fs_type} {options_line} 0 0") - - fstab = Path("/etc/fstab") - if not Path(fstab.with_suffix(".bak")).is_file(): - shutil.copy2(fstab, fstab.with_suffix(".bak")) - shutil.copy2(fstab.with_suffix(".bak"), fstab) - with open(fstab, "a") as f: - f.write("\n") - for entry in fstab_entries: - f.write(entry) - f.write("\n") - - mount_fstab(mounts, log) - if lookup().cfg.enable_slurm_auth: - slurm_key_mount_handler() - else: - munge_mount_handler() - - -def mount_fstab(mounts: list[NSMount], log): - """Wait on each mount, then make sure all fstab is mounted""" - def mount_path(path: Path): - log.info(f"Waiting for '{path}' to be mounted...") - try: - run(f"mount {path}", timeout=120) - except Exception as e: - exc_type, _, _ = sys.exc_info() - log.error(f"mount of path '{path}' failed: {exc_type}: {e}") - raise e - log.info(f"Mount point '{path}' was mounted.") - - MAX_MOUNT_TIMEOUT = 60 * 5 - future_list = [] - retry_policy = ExceptionRetryPolicy( - max_attempts=40, exponent=1.6, sleep=1.0, max_sleep=16.0 - ) - with Executors.thread_pool().with_timeout(MAX_MOUNT_TIMEOUT).with_retry( - retry_policy=retry_policy - ) as exe: - for m in mounts: - future = exe.submit(mount_path, m.local_mount) - future_list.append(future) - - # Iterate over futures, checking for exceptions - for future in as_completed(future_list): - try: - future.result() - except Exception as e: - raise e - - -def munge_mount_handler(): - if lookup().is_controller: - return - mnt = lookup().munge_mount - - log.info(f"Mounting munge share to: {mnt.local_mount}") - mnt.local_mount.mkdir() - if mnt.fs_type == "gcsfuse": - cmd = [ - "gcsfuse", - f"--only-dir={mnt.remote_mount}" if mnt.remote_mount != "" else None, - mnt.server_ip, - str(mnt.local_mount), - ] - else: - cmd = [ - "mount", - f"--types={mnt.fs_type}", - f"--options={mnt.mount_options}" if mnt.mount_options != "" else None, - f"{mnt.server_ip}:{mnt.remote_mount}", - str(mnt.local_mount), - ] - # wait max 120s for munge mount - timeout = 120 - for retry, wait in enumerate(util.backoff_delay(0.5, timeout), 1): - try: - run(cmd, timeout=timeout) - break - except Exception as e: - log.error( - f"munge mount failed: '{cmd}' {e}, try {retry}, waiting {wait:0.2f}s" - ) - time.sleep(wait) - err = e - continue - else: - raise err - - munge_key = Path(dirs.munge / "munge.key") - log.info(f"Copy munge.key from: {mnt.local_mount}") - shutil.copy2(Path(mnt.local_mount / "munge.key"), munge_key) - - log.info("Restrict permissions of munge.key") - shutil.chown(munge_key, user="munge", group="munge") - os.chmod(munge_key, stat.S_IRUSR) - - log.info(f"Unmount {mnt.local_mount}") - if mnt.fs_type == "gcsfuse": - run(f"fusermount -u {mnt.local_mount}", timeout=120) - else: - run(f"umount {mnt.local_mount}", timeout=120) - shutil.rmtree(mnt.local_mount) - -def slurm_key_mount_handler(): - if lookup().is_controller: - return - mnt = lookup().slurm_key_mount - - log.info(f"Mounting slurm_key share to: {mnt.local_mount}") - if mnt.fs_type == "gcsfuse": - cmd = [ - "gcsfuse", - f"--only-dir={mnt.remote_mount}" if mnt.remote_mount != "" else None, - mnt.server_ip, - str(mnt.local_mount), - ] - else: - cmd = [ - "mount", - f"--types={mnt.fs_type}", - f"--options={mnt.mount_options}" if mnt.mount_options != "" else None, - f"{mnt.server_ip}:{mnt.remote_mount}", - str(mnt.local_mount), - ] - timeout = 120 # wait max 120s to mount - for retry, wait in enumerate(util.backoff_delay(0.5, timeout), 1): - try: - run(cmd, timeout=timeout) - break - except Exception as e: - log.error( - f"slurm key mount failed: '{cmd}' {e}, try {retry}, waiting {wait:0.2f}s" - ) - time.sleep(wait) - err = e - continue - else: - raise err - - file_name = "slurm.key" - dst = Path(util.slurmdirs.etc / file_name) - log.info(f"Copy slurm.key from: {mnt.local_mount}") - shutil.copy2(mnt.local_mount / file_name, dst) - - log.info("Restrict permissions of slurm.key") - util.chown_slurm(dst, mode=0o400) - - log.info(f"Unmount {mnt.local_mount}") - if mnt.fs_type == "gcsfuse": - run(f"fusermount -u {mnt.local_mount}", timeout=120) - else: - run(f"umount {mnt.local_mount}", timeout=120) - shutil.rmtree(mnt.local_mount) - - -def setup_nfs_exports(): - """nfs export all needed directories""" - lkp = util.lookup() - assert lkp.is_controller - - # The controller only needs to set up exports for cluster-internal mounts - exported_mounts = [m for m in resolve_network_storage() if is_controller_mount(m)] - - # key by remote mount path since that is what needs exporting - to_export = {m.remote_mount: "*(rw,no_subtree_check,no_root_squash)" for m in exported_mounts} - - key_mount = lkp.slurm_key_mount if lkp.cfg.enable_slurm_auth else lkp.munge_mount - if is_controller_mount(key_mount): - # Export key mount as read-only - to_export[key_mount.remote_mount] = "*(ro,no_subtree_check,no_root_squash)" - - if util.should_mount_slurm_bucket(): - mnt = get_slurm_bucket_mount() - # FSID is required for virtual filesystem that is not based on a device - # Also export it as read-only - fsid=str(uuid.uuid4()) - to_export[mnt.local_mount] = f"*(ro,no_subtree_check,no_root_squash,fsid={fsid})" - - # export path if corresponding selector boolean is True - lines = [] - for path,options in to_export.items(): - util.mkdirp(Path(path)) - run(rf"sed -i '\#{path}#d' /etc/exports", timeout=30) - lines.append(f"{path} {options}") - - exportsd = Path("/etc/exports.d") - util.mkdirp(exportsd) - with (exportsd / "slurm.exports").open("w") as f: - f.write("\n") - f.write("\n".join(lines)) - run("exportfs -a", timeout=30) diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py deleted file mode 100644 index 7285096c7c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/slurmsync.py +++ /dev/null @@ -1,661 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import fcntl -import json -import logging -import re -import sys -import shlex -from datetime import datetime, timedelta -from itertools import chain -from pathlib import Path -from dataclasses import dataclass -from typing import Dict, Tuple, List, Optional, Protocol, Any -from functools import lru_cache - -import util -from util import ( - batch_execute, - ensure_execute, - execute_with_futures, - FutureReservation, - install_custom_scripts, - run, - separate, - to_hostlist, - NodeState, - chunked, - dirs, -) -from util import lookup -from suspend import delete_instances -import tpu -import conf -import watch_delete_vm_op - -log = logging.getLogger() - -TOT_REQ_CNT = 1000 -_MAINTENANCE_SBATCH_SCRIPT_PATH = dirs.custom_scripts / "perform_maintenance.sh" - -class NodeAction(Protocol): - def apply(self, nodes:List[str]) -> None: - ... - - def __hash__(self): - ... - -@dataclass(frozen=True) -class NodeActionPowerUp(): - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} instances to resume ({hostlist})") - run(f"{lookup().scontrol} update nodename={hostlist} state=power_up") - -@dataclass(frozen=True) -class NodeActionIdle(): - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} nodes to idle ({hostlist})") - run(f"{lookup().scontrol} update nodename={hostlist} state=resume") - -@dataclass(frozen=True) -class NodeActionPowerDown(): - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} instances to power down ({hostlist})") - run(f"{lookup().scontrol} update nodename={hostlist} state=power_down") - -@dataclass(frozen=True) -class NodeActionDelete(): - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} instances to delete ({hostlist})") - delete_instances(nodes) - -@dataclass(frozen=True) -class NodeActionPrempt(): - def apply(self, nodes:List[str]) -> None: - NodeActionDown(reason="Preempted instance").apply(nodes) - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} instances restarted ({hostlist})") - start_instances(nodes) - -@dataclass(frozen=True) -class NodeActionUnchanged(): - def apply(self, nodes:List[str]) -> None: - pass - -@dataclass(frozen=True) -class NodeActionDown(): - reason: str - - def apply(self, nodes: List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.info(f"{len(nodes)} nodes set down ({hostlist}) with reason={self.reason}") - run(f"{lookup().scontrol} update nodename={hostlist} state=down reason={shlex.quote(self.reason)}") - -@dataclass(frozen=True) -class NodeActionUnknown(): - slurm_state: Optional[NodeState] - instance_state: Optional[str] - - def apply(self, nodes:List[str]) -> None: - hostlist = util.to_hostlist(nodes) - log.error(f"{len(nodes)} nodes have unexpected {self.slurm_state} and instance state:{self.instance_state}, ({hostlist})") - -def start_instance_op(node: str) -> Any: - inst = lookup().instance(node) - assert inst - - return lookup().compute.instances().start( - project=lookup().project, - zone=inst.zone, - instance=inst, - ) - - -def start_instances(node_list): - log.info("{} instances to start ({})".format(len(node_list), ",".join(node_list))) - lkp = lookup() - # TODO: use code from resume.py to assign proper placement - normal, tpu_nodes = separate(lkp.node_is_tpu, node_list) - ops = {node: start_instance_op(node) for node in normal} - - done, failed = batch_execute(ops) - - tpu_start_data = [] - for ns, nodes in util.groupby_unsorted(tpu_nodes, lkp.node_nodeset_name): - tpuobj = tpu.TPU.make(ns, lkp) - for snodes in chunked(nodes, n=tpuobj.vmcount): - tpu_start_data.append({"tpu": tpuobj, "node": snodes}) - execute_with_futures(tpu.start_tpu, tpu_start_data) - - -def _find_dynamic_node_status() -> NodeAction: - # TODO: cover more cases: - # * delete dead dynamic nodes - # * delete orhpaned instances - return NodeActionUnchanged() # don't touch dynamic nodes - -def get_fr_action(fr: FutureReservation, state:Optional[NodeState]) -> Optional[NodeAction]: - now = util.now() - if state is None: - return None # handle like any other node - if fr.start_time < now < fr.end_time: - return None # handle like any other node - - if state.base == "DOWN": - return NodeActionUnchanged() - if fr.start_time >= now: - msg = f"Waiting for reservation:{fr.name} to start at {fr.start_time}" - else: - msg = f"Reservation:{fr.name} is after its end-time" - return NodeActionDown(reason=msg) - -def _find_tpu_node_action(nodename, state) -> NodeAction: - lkp = lookup() - tpuobj = tpu.TPU.make(lkp.node_nodeset_name(nodename), lkp) - inst = tpuobj.get_node(nodename) - # If we do not find the node but it is from a Tpu that has multiple vms look for the master node - if inst is None and tpuobj.vmcount > 1: - # Get the tpu slurm nodelist of the nodes in the same tpu group as nodename - nodelist = run( - f"{lkp.scontrol} show topo {nodename}" - + " | awk -F'=' '/Level=0/ { print $NF }'", - shell=True, - ).stdout - l_nodelist = util.to_hostnames(nodelist) - group_names = set(l_nodelist) - # get the list of all the existing tpus in the nodeset - tpus_list = set(tpuobj.list_node_names()) - # In the intersection there must be only one node that is the master - tpus_int = list(group_names.intersection(tpus_list)) - if len(tpus_int) > 1: - log.error( - f"More than one cloud tpu node for tpu group {nodelist}, there should be only one that should be {l_nodelist[0]}, but we have found {tpus_int}" - ) - return NodeActionUnknown(slurm_state=state, instance_state=None) - if len(tpus_int) == 1: - inst = tpuobj.get_node(tpus_int[0]) - # if len(tpus_int ==0) this case is not relevant as this would be the case always that a TPU group is not running - if inst is None: - if state.base == "DOWN" and "POWERED_DOWN" in state.flags: - return NodeActionIdle() - if "POWERING_DOWN" in state.flags: - return NodeActionIdle() - if "COMPLETING" in state.flags: - return NodeActionDown(reason="Unbacked instance") - if state.base != "DOWN" and not ( - set(("POWER_DOWN", "POWERING_UP", "POWERING_DOWN", "POWERED_DOWN")) - & state.flags - ): - return NodeActionDown(reason="Unbacked instance") - if lkp.is_static_node(nodename): - return NodeActionPowerUp() - elif ( - state is not None - and "POWERED_DOWN" not in state.flags - and "POWERING_DOWN" not in state.flags - and inst.state == tpu.TPU.State.STOPPED - ): - if tpuobj.preemptible: - return NodeActionPrempt() - if state.base != "DOWN": - return NodeActionDown(reason="Instance terminated") - elif ( - state is None or "POWERED_DOWN" in state.flags - ) and inst.state == tpu.TPU.State.READY: - return NodeActionDelete() - elif state is None: - # if state is None here, the instance exists but it's not in Slurm - return NodeActionUnknown(slurm_state=state, instance_state=inst.status) - - return NodeActionUnchanged() - -def get_node_action(nodename: str) -> NodeAction: - """Determine node/instance status that requires action""" - lkp = lookup() - state = lkp.node_state(nodename) - - if lkp.node_is_fr(nodename): - fr = lkp.future_reservation(lkp.node_nodeset(nodename)) - assert fr - if action := get_fr_action(fr, state): - return action - - if lkp.node_is_dyn(nodename): - return _find_dynamic_node_status() - - if lkp.node_is_tpu(nodename): - return _find_tpu_node_action(nodename, state) - - # split below is workaround for VMs whose hostname is FQDN - inst = lkp.instance(nodename.split(".")[0]) - power_flags = frozenset( - ("POWER_DOWN", "POWERING_UP", "POWERING_DOWN", "POWERED_DOWN") - ) & (state.flags if state is not None else set()) - - if (state is None) and (inst is None): - # Should never happen - return NodeActionUnknown(None, None) - if inst is None: - assert state is not None # to keep type-checker happy - if "POWERING_UP" in state.flags: - return NodeActionUnchanged() - if state.base == "DOWN" and "POWERED_DOWN" in state.flags: - return NodeActionIdle() - if "POWERING_DOWN" in state.flags: - return NodeActionIdle() - if "COMPLETING" in state.flags: - return NodeActionDown(reason="Unbacked instance") - if state.base != "DOWN" and not power_flags: - return NodeActionDown(reason="Unbacked instance") - if state.base == "DOWN" and not power_flags: - return NodeActionPowerDown() - if "POWERED_DOWN" in state.flags and lkp.is_static_node(nodename): - return NodeActionPowerUp() - elif ( - state is not None - and "POWERED_DOWN" not in state.flags - and "POWERING_DOWN" not in state.flags - and inst.status == "TERMINATED" - ): - if inst.scheduling.preemptible: - return NodeActionPrempt() - if state.base != "DOWN": - return NodeActionDown(reason="Instance terminated") - elif (state is None or "POWERED_DOWN" in state.flags) and inst.status == "RUNNING": - log.info("%s is potential orphan node", nodename) - threshold = timedelta(seconds=90) - age = util.now() - inst.creation_timestamp - log.info(f"{nodename} state: {state}, age: {age}") - if age < threshold: - log.info(f"{nodename} not marked as orphan, it started less than {threshold.seconds}s ago ({age.seconds}s)") - return NodeActionUnchanged() - return NodeActionDelete() - elif state is None: - # if state is None here, the instance exists but it's not in Slurm - return NodeActionUnknown(slurm_state=state, instance_state=inst.status) - - return NodeActionUnchanged() - - -def delete_resource_policies(links: list[str], lkp: util.Lookup) -> None: - requests = {} - for link in links: - name = util.trim_self_link(link) - region = util.parse_self_link(link).region - requests[name] = lkp.compute.resourcePolicies().delete(project=lkp.project, region=region, resourcePolicy=name) - - def swallow_err(_: str) -> None: - pass - - done, failed = batch_execute(requests, log_err=swallow_err) - if failed: - # Filter out resourceInUseByAnotherResource errors , they are expected to happen - def ignore_err(e) -> bool: - return "resourceInUseByAnotherResource" in str(e) - - failures = [f"{n}: {e}" for n, (_, e) in failed.items() if not ignore_err(e)] - if failures: - log.error(f"some placement groups failed to delete: {failures}") - log.info( - f"deleted {len(done)} of {len(links)} placement groups ({to_hostlist(done.keys())})" - ) - - - -@lru_cache -def _get_resource_policies_in_region(lkp: util.Lookup, region: str) -> list[Any]: - res = [] - act = lkp.compute.resourcePolicies() - op = act.list(project=lkp.project, region=region) - prefix = f"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-" - while op is not None: - result = ensure_execute(op) - res.extend([p for p in result.get("items", []) if p.get("name", "").startswith(prefix)]) - op = act.list_next(op, result) - return res - - -@lru_cache -def _get_resource_policies(lkp: util.Lookup) -> list[Any]: - res = [] - for region in lkp.cluster_regions(): - res.extend(_get_resource_policies_in_region(lkp, region)) - return res - -def sync_placement_groups(): - """Delete placement policies that are for jobs that have completed/terminated""" - keep_states = frozenset( - [ - "RUNNING", - "CONFIGURING", - "STOPPED", - "SUSPENDED", - "COMPLETING", - "PENDING", - ] - ) - - lkp = lookup() - keep_jobs = { - str(job.id) - for job in lkp.get_jobs() - if job.job_state in keep_states - } - keep_jobs.add("0") # Job 0 is a placeholder for static node placement - - to_delete = [] - pg_regex = re.compile( - rf"{lkp.cfg.slurm_cluster_name}-slurmgcp-managed-(?P[^\s\-]+)-(?P\d+)-(?P\d+)" - ) - - for pg in _get_resource_policies(lkp): - name = pg["name"] - - if (mtch := pg_regex.match(name)) is None: - log.warning(f"Unexpected resource policy {name=}") - continue - if mtch.group("job_id") not in keep_jobs: - to_delete.append(pg["selfLink"]) - - if to_delete: - delete_resource_policies(to_delete, lkp) - - -def sync_instances(): - compute_instances = { - name for name, inst in lookup().instances().items() if inst.role == "compute" - } - slurm_nodes = set(lookup().slurm_nodes().keys()) - log.debug(f"reconciling {len(compute_instances)} GCP instances and {len(slurm_nodes)} Slurm nodes.") - - for action, nodes in util.groupby_unsorted(list(compute_instances | slurm_nodes), get_node_action): - action.apply(list(nodes)) - - -def reconfigure_slurm(): - update_msg = "*** slurm configuration was updated ***" - if lookup().cfg.hybrid: - # terraform handles generating the config.yaml, don't do it here - return - - upd, cfg_new = util.fetch_config() - if not upd: - log.debug("No changes in config detected.") - return - log.debug("Changes in config detected. Reconfiguring Slurm now.") - util.update_config(cfg_new) - - if lookup().is_controller: - conf.gen_controller_configs(lookup()) - log.info("Restarting slurmctld to make changes take effect.") - try: - # TODO: consider removing "restart" since "reconfigure" should restart slurmctld as well - run("sudo systemctl restart slurmctld.service", check=False) - util.scontrol_reconfigure(lookup()) - except Exception: - log.exception("failed to reconfigure slurmctld") - util.run(f"wall '{update_msg}'", timeout=30) - log.debug("Done.") - elif lookup().instance_role_safe == "compute": - log.info("Restarting slurmd to make changes take effect.") - run("systemctl restart slurmd") - util.run(f"wall '{update_msg}'", timeout=30) - log.debug("Done.") - elif lookup().is_login_node: - log.info("Restarting sackd to make changes take effect.") - run("systemctl restart sackd") - util.run(f"wall '{update_msg}'", timeout=30) - log.debug("Done.") - - -def update_topology(lkp: util.Lookup) -> None: - if conf.topology_plugin(lkp) != conf.TOPOLOGY_PLUGIN_TREE: - return - updated, summary = conf.gen_topology_conf(lkp) - if updated: - log.info("Topology configuration updated. Reconfiguring Slurm.") - util.scontrol_reconfigure(lkp) - # Safe summary only after Slurm got reconfigured, so summary reflects Slurm POV - summary.dump(lkp) - - -def delete_reservation(lkp: util.Lookup, reservation_name: str) -> None: - util.run(f"{lkp.scontrol} delete reservation {reservation_name}") - - -def create_reservation(lkp: util.Lookup, reservation_name: str, node: str, start_time: datetime) -> None: - # Format time to be compatible with slurm reservation. - formatted_start_time = start_time.strftime('%Y-%m-%dT%H:%M:%S') - - util.run(f"{lkp.scontrol} create reservation user=slurm starttime={formatted_start_time} duration=180 nodes={node} reservationname={reservation_name} flags=maint,ignore_jobs") - - -def get_slurm_reservation_maintenance(lkp: util.Lookup) -> Dict[str, datetime]: - res = util.run(f"{lkp.scontrol} show reservation --json") - all_reservations = json.loads(res.stdout) - reservation_map = {} - - for reservation in all_reservations['reservations']: - name = reservation.get('name') - nodes = reservation.get('node_list') - time_epoch = reservation.get('start_time', {}).get('number') - - if name is None or nodes is None or time_epoch is None: - continue - - if reservation.get('node_count') != 1: - continue - - if name != f"{nodes}_maintenance": - continue - - reservation_map[name] = datetime.fromtimestamp(time_epoch) - - return reservation_map - -@lru_cache -def get_upcoming_maintenance(lkp: util.Lookup) -> Dict[str, Tuple[str, datetime]]: - upc_maint_map = {} - - for node, inst in lkp.instances().items(): - if inst.resource_status.upcoming_maintenance: - upc_maint_map[node + "_maintenance"] = (node, inst.resource_status.upcoming_maintenance.window_start_time) - - return upc_maint_map - - -def sync_maintenance_reservation(lkp: util.Lookup) -> None: - upc_maint_map = get_upcoming_maintenance(lkp) # map reservation_name -> (node_name, time) - log.debug(f"upcoming-maintenance-vms: {upc_maint_map}") - - curr_reservation_map = get_slurm_reservation_maintenance(lkp) # map reservation_name -> time - log.debug(f"curr-reservation-map: {curr_reservation_map}") - - del_reservation = set(curr_reservation_map.keys() - upc_maint_map.keys()) - create_reservation_map = {} - - for res_name, (node, start_time) in upc_maint_map.items(): - try: - enabled = lkp.node_nodeset(node).enable_maintenance_reservation - except Exception: - enabled = False - - if not enabled: - if res_name in curr_reservation_map: - del_reservation.add(res_name) - continue - - if res_name in curr_reservation_map: - diff = curr_reservation_map[res_name] - start_time - if abs(diff) <= timedelta(seconds=1): - continue - else: - del_reservation.add(res_name) - create_reservation_map[res_name] = (node, start_time) - else: - create_reservation_map[res_name] = (node, start_time) - - log.debug(f"del-reservation: {del_reservation}") - for res_name in del_reservation: - delete_reservation(lkp, res_name) - - log.debug(f"create-reservation-map: {create_reservation_map}") - for res_name, (node, start_time) in create_reservation_map.items(): - create_reservation(lkp, res_name, node, start_time) - - -def delete_maintenance_job(job_name: str) -> None: - util.run(f"scancel --name={job_name}") - - -def create_maintenance_job(job_name: str, node: str) -> None: - util.run(f"sbatch --job-name={job_name} --nodelist={node} {_MAINTENANCE_SBATCH_SCRIPT_PATH}") - - -def get_slurm_maintenance_job(lkp: util.Lookup) -> Dict[str, str]: - jobs = {} - - for job in lkp.get_jobs(): - if job.name is None or job.required_nodes is None or job.job_state is None: - continue - - if job.name != f"{job.required_nodes}_maintenance": - continue - - if job.job_state != "PENDING": - continue - - jobs[job.name] = job.required_nodes - - return jobs - - -def sync_opportunistic_maintenance(lkp: util.Lookup) -> None: - upc_maint_map = get_upcoming_maintenance(lkp) # map job_name -> (node_name, time) - log.debug(f"upcoming-maintenance-vms: {upc_maint_map}") - - curr_jobs = get_slurm_maintenance_job(lkp) # map job_name -> node. - log.debug(f"curr-maintenance-job-map: {curr_jobs}") - - del_jobs = set(curr_jobs.keys() - upc_maint_map.keys()) - create_jobs = {} - - for job_name, (node, _) in upc_maint_map.items(): - try: - enabled = lkp.node_nodeset(node).enable_opportunistic_maintenance - except Exception: - enabled = False - - if not enabled: - if job_name in curr_jobs: - del_jobs.add(job_name) - continue - - if job_name not in curr_jobs: - create_jobs[job_name] = node - - log.debug(f"del-maintenance-job: {del_jobs}") - for job_name in del_jobs: - delete_maintenance_job(job_name) - - log.debug(f"create-maintenance-job: {create_jobs}") - for job_name, node in create_jobs.items(): - create_maintenance_job(job_name, node) - - - -def sync_flex_migs(lkp: util.Lookup) -> None: - pass - - -def process_messages(lkp: util.Lookup) -> None: - try: - watch_delete_vm_op.watch_vm_delete_ops(lkp) - except: - log.exception("failed during watching delete VM operations") - - -def main(): - lkp = lookup() - if util.should_mount_slurm_bucket() and not lkp.is_controller: - return - try: - reconfigure_slurm() - except Exception: - log.exception("failed to reconfigure slurm") - if lkp.is_controller: - try: - process_messages(lkp) - except: - log.exception("failed to process messages") - - try: - sync_instances() - except Exception: - log.exception("failed to sync instances") - - try: - sync_flex_migs(lkp) - except Exception: - log.exception("failed to sync DWS Flex MIGs") - - try: - sync_placement_groups() - except Exception: - log.exception("failed to sync placement groups") - - try: - update_topology(lkp) - except Exception: - log.exception("failed to update topology") - - try: - sync_maintenance_reservation(lkp) - except Exception: - log.exception("failed to sync slurm reservation for scheduled maintenance") - - try: - sync_opportunistic_maintenance(lkp) - except Exception: - log.exception("failed to sync opportunistic reservation for scheduled maintenance") - - - try: - # TODO: it performs 1 to 4 GCS list requests, - # use cached version, combine with `_list_config_blobs` - install_custom_scripts(check_hash=True) - except Exception: - log.exception("failed to sync custom scripts") - - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - _ = util.init_log_and_parse(parser) - - pid_file = (Path("/tmp") / Path(__file__).name).with_suffix(".pid") - with pid_file.open("w") as fp: - try: - fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB) - main() - except BlockingIOError: - sys.exit(0) diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py deleted file mode 100644 index ae36c54222..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/sort_nodes.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -This script sorts nodes based on their `physicalHost`. - -See https://cloud.google.com/compute/docs/instances/use-compact-placement-policies - -You can reduce latency in tightly coupled HPC workloads (including distributed ML training) -by deploying them to machines that are located close together. -For example, if you deploy your workload on a single physical rack, you can expect lower latency -than if your workload is spread across multiple racks. -Sending data across multiple rack requires sending data through additional network switches. - -Example usage: -``` my_sbatch.sh -#SBATCH --ntasks-per-node=8 -#SBATCH --nodes=64 - -export SLURM_HOSTFILE=$(sort_nodes.py) - -srun -l hostname | sort -``` -""" -import os -import subprocess -import uuid -from typing import List, Optional, Dict -from collections import OrderedDict - -def order(paths: List[List[str]]) -> List[str]: - """ - Orders the leaves of the tree in a way that minimizes the sum of distance in between - each pair of neighboring nodes in the resulting order. - The resulting order will always start from the first node in the input list. - The ordering is "stable" with respect to the input order of the leaves i.e. - given a choice between two nodes (identical in other ways) it will select "nodelist-smallest" one. - - Returns a list of nodenames, ordered as described above. - """ - if not paths: return [] - class Vert: - "Represents a vertex in a *network* tree." - def __init__(self, name: str, parent: Optional["Vert"]): - self.name = name - self.parent = parent - # Use `OrderedDict` to preserve insertion order - # TODO: once we move to Python 3.7+ use regular `dict` since it has the same guarantee - self.children: OrderedDict = OrderedDict() - - # build a tree, children are ordered by insertion order - root = Vert("", None) - for path in paths: - n = root - for v in path: - if v not in n.children: - n.children[v] = Vert(v, n) - n = n.children[v] - - # walk the tree in insertion order, gather leaves - result = [] - def gather_nodes(v: Vert) -> None: - if not v.children: # this is a Slurm node - result.append(v.name) - for u in v.children.values(): - gather_nodes(u) - gather_nodes(root) - return result - - -class Instance: - def __init__(self, name: str, zone: str, physical_host: Optional[str]): - self.name = name - self.zone = zone - self.physical_host = physical_host - - -def make_path(node_name: str, inst: Optional[Instance]) -> List[str]: - if not inst: # node with unknown instance (e.g. hybrid cluster) - return ["unknown", node_name] - zone = f"zone_{inst.zone}" - if not inst.physical_host: # node without physical host info (e.g. no placement policy) - return [zone, "unknown", node_name] - - assert inst.physical_host.startswith("/"), f"Unexpected physicalHost: {inst.physical_host}" - parts = inst.physical_host[1:].split("/") - if len(parts) >= 4: - return [*parts, node_name] - return [zone, *parts, node_name] - - -def to_hostnames(nodelist: str) -> List[str]: - cmd = ["scontrol", "show", "hostnames", nodelist] - out = subprocess.run(cmd, check=True, stdout=subprocess.PIPE).stdout - return [n.decode("utf-8") for n in out.splitlines()] - - -def get_instances(node_names: List[str]) -> Dict[str, Optional[Instance]]: - fmt = ( - "--format=csv[no-heading,separator=','](zone,resourceStatus.physicalHost,name)" - ) - cmd = ["gcloud", "compute", "instances", "list", fmt] - - scp = os.path.commonprefix(node_names) - if scp: - cmd.append(f"--filter=name~'{scp}.*'") - out = subprocess.run(cmd, check=True, stdout=subprocess.PIPE).stdout - d = {} - for line in out.splitlines(): - zone, physical_host, name = line.decode("utf-8").split(",") - d[name] = Instance(name, zone, physical_host) - return {n: d.get(n) for n in node_names} - - -def main(args) -> None: - nodelist = args.nodelist or os.getenv("SLURM_NODELIST") - if not nodelist: - raise ValueError("nodelist is not provided and SLURM_NODELIST is not set") - - if args.ntasks_per_node is None: - args.ntasks_per_node = int(os.getenv("SLURM_NTASKS_PER_NODE", "") or 1) - assert args.ntasks_per_node > 0 - - output = args.output or f"hosts.{uuid.uuid4()}" - - node_names = to_hostnames(nodelist) - instannces = get_instances(node_names) - paths = [make_path(n, instannces[n]) for n in node_names] - ordered = order(paths) - - with open(output, "w") as f: - for node in ordered: - for _ in range(args.ntasks_per_node): - f.write(node) - f.write("\n") - print(output) - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument( - "--nodelist", - type=str, - help="Slurm 'hostlist expression' of nodes to sort, if not set the value of SLURM_NODELIST environment variable will be used", - ) - parser.add_argument( - "--ntasks-per-node", - type=int, - help="""Number of times to repeat each node in resulting sorted list. -If not set, the value of SLURM_NTASKS_PER_NODE environment variable will be used, -if neither is set, defaults to 1""", - ) - parser.add_argument( - "--output", type=str, help="Output file to write, defaults to 'hosts.'" - ) - args = parser.parse_args() - main(args) diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py deleted file mode 100644 index ecef70f1cc..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List, Any -import argparse -import logging - -import util -from util import ( - log_api_request, - batch_execute, - to_hostlist, - separate, -) -from util import lookup -import tpu -import mig_flex -import watch_delete_vm_op - -log = logging.getLogger() - -TOT_REQ_CNT = 1000 - - -def truncate_iter(iterable, max_count): - end = "..." - _iter = iter(iterable) - for i, el in enumerate(_iter, start=1): - if i >= max_count: - yield end - break - yield el - - -def delete_instance_request(name: str) -> Any: - inst = lookup().instance(name) - assert inst - - request = lookup().compute.instances().delete( - project=lookup().project, - zone=inst.zone, - instance=name, - ) - log_api_request(request) - return request - - -def delete_instances(instances): - """delete instances individually""" - invalid, valid = separate(lambda inst: bool(lookup().instance(inst)), instances) - if len(invalid) > 0: - log.debug("instances do not exist: {}".format(",".join(invalid))) - if len(valid) == 0: - log.debug("No instances to delete") - return - - requests = {inst: delete_instance_request(inst) for inst in valid} - - log.info(f"to delete {len(valid)} instances ({to_hostlist(valid)})") - ops, failed = batch_execute(requests) - for node, (_, err) in failed.items(): - log.error(f"instance {node} failed to delete: {err}") - - log.info(f"deleting {len(ops)} instances {to_hostlist(ops.keys())}") - - topic = watch_delete_vm_op.watch_delete_vm_op_topic() - for node, op in ops.items(): - topic.publish(op, node) - - - - -def suspend_nodes(nodes: List[str]) -> None: - lkp = lookup() - other_nodes, tpu_nodes = util.separate(lkp.node_is_tpu, nodes) - bulk_nodes, flex_nodes = util.separate(lkp.is_flex_node, other_nodes) - - mig_flex.suspend_flex_nodes(flex_nodes, lkp) - delete_instances(bulk_nodes) - tpu.delete_tpu_instances(tpu_nodes) - - -def main(nodelist): - """main called when run as script""" - log.debug(f"SuspendProgram {nodelist}") - - # Filter out nodes not in config.yaml - other_nodes, pm_nodes = separate( - lookup().is_power_managed_node, util.to_hostnames(nodelist) - ) - if other_nodes: - log.debug( - f"Ignoring non-power-managed nodes '{to_hostlist(other_nodes)}' from '{nodelist}'" - ) - if pm_nodes: - log.debug(f"Suspending nodes '{to_hostlist(pm_nodes)}' from '{nodelist}'") - else: - log.debug("No cloud nodes to suspend") - return - - log.info(f"suspend {nodelist}") - suspend_nodes(pm_nodes) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - parser.add_argument("nodelist", help="list of nodes to suspend") - args = util.init_log_and_parse(parser) - - main(args.nodelist) diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh deleted file mode 100644 index 9079e4e4b0..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/suspend_wrapper.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) -PYTHON_SCRIPT="${SCRIPT_DIR}/suspend.py" - -# Capture all arguments passed by Slurm (the nodelist). -ALL_ARGS=("$@") - -"${PYTHON_SCRIPT}" "${ALL_ARGS[@]}" & -disown - -exit 0 diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py deleted file mode 100644 index ea4b07c64d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/common.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional, Any -import sys -from dataclasses import dataclass, field -from datetime import datetime - -SCRIPTS_DIR = "community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts" -if SCRIPTS_DIR not in sys.path: - sys.path.append(SCRIPTS_DIR) # TODO: make this more robust - -import util - - -SOME_TS = datetime.fromisoformat("2018-09-03T20:56:35.450686+00:00") -# TODO: use "real" classes once they are defined (instead of NSDict) - -@dataclass -class Placeholder: - pass - -@dataclass -class TstNodeset: - nodeset_name: str = "cantor" - node_count_static: int = 0 - node_count_dynamic_max: int = 0 - node_conf: dict[str, Any] = field(default_factory=dict) - instance_template: Optional[str] = None - reservation_name: Optional[str] = "" - zone_policy_allow: Optional[list[str]] = field(default_factory=list) - enable_placement: bool = True - placement_max_distance: Optional[int] = None - future_reservation: Optional[str] = "" - -@dataclass -class TstPartition: - partition_name: str = "euler" - partition_nodeset: list[str] = field(default_factory=list) - partition_nodeset_tpu: list[str] = field(default_factory=list) - enable_job_exclusive: bool = False - -@dataclass -class TstCfg: - slurm_cluster_name: str = "m22" - cloud_parameters: dict[str, Any] = field(default_factory=dict) - - partitions: dict[str, TstPartition] = field(default_factory=dict) - nodeset: dict[str, TstNodeset] = field(default_factory=dict) - nodeset_tpu: dict[str, TstNodeset] = field(default_factory=dict) - nodeset_dyn: dict[str, TstNodeset] = field(default_factory=dict) - - install_dir: Optional[str] = None - output_dir: Optional[str] = None - - prolog_scripts: Optional[list[Placeholder]] = field(default_factory=list) - epilog_scripts: Optional[list[Placeholder]] = field(default_factory=list) - task_prolog_scripts: Optional[list[Placeholder]] = field(default_factory=list) - task_epilog_scripts: Optional[list[Placeholder]] = field(default_factory=list) - - -@dataclass -class TstTPU: # to prevent client initialization durint "TPU.__init__" - vmcount: int - -@dataclass -class TstMachineConf: - cpus: int - memory: int - sockets: int - sockets_per_board: int - cores_per_socket: int - boards: int - threads_per_core: int - - -@dataclass -class TstTemplateInfo: - gpu: Optional[util.AcceleratorInfo] - -def tstInstance(name: str, physical_host: Optional[str] = None): - return util.Instance( - name=name, - zone="anorien", - status="RUNNING", - creation_timestamp=SOME_TS, - resource_status=util.InstanceResourceStatus( - physical_host=physical_host, - upcoming_maintenance=None, - ), - scheduling=util.NSDict(), - role="compute", - metadata={}, - ) - -def make_to_hostnames_mock(tbl: Optional[dict[str, list[str]]]): - tbl = tbl or {} - - def se(k: str) -> list[str]: - if k not in tbl: - raise AssertionError(f"to_hostnames mock: unexpected nodelist: '{k}'") - return tbl[k] - - return se diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py deleted file mode 100644 index 5fb4df5e82..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_conf.py +++ /dev/null @@ -1,192 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -from mock import Mock -from common import TstNodeset, TstCfg, TstMachineConf, TstTemplateInfo, Placeholder - -import addict # type: ignore -import conf -import util - - -def test_nodeset_tpu_lines(): - nodeset = TstNodeset( - "turbo", - node_count_static=2, - node_count_dynamic_max=3, - node_conf={"red": "velvet"}, - ) - assert conf.nodeset_tpu_lines(nodeset, util.Lookup(TstCfg())) == "\n".join( - [ - "NodeName=m22-turbo-[0-4] State=CLOUD red=velvet", - "NodeSet=turbo Nodes=m22-turbo-[0-4]", - ] - ) - - -def test_nodeset_lines(): - nodeset = TstNodeset( - "turbo", - node_count_static=2, - node_count_dynamic_max=3, - node_conf={"red": "velvet", "CPUs": 55}, - ) - lkp = util.Lookup(TstCfg()) - lkp.template_info = Mock(return_value=TstTemplateInfo( - gpu=util.AcceleratorInfo(type="Popov", count=33) - )) - mc = TstMachineConf( - cpus=5, - memory=6, - sockets=7, - sockets_per_board=8, - boards=9, - threads_per_core=10, - cores_per_socket=11, - ) - lkp.template_machine_conf = Mock(return_value=mc) # type: ignore[method-assign] - assert conf.nodeset_lines(nodeset, lkp) == "\n".join( - [ - "NodeName=m22-turbo-[0-4] State=CLOUD RealMemory=6 Boards=9 SocketsPerBoard=8 CoresPerSocket=11 ThreadsPerCore=10 CPUs=55 Gres=gpu:33 red=velvet", - "NodeSet=turbo Nodes=m22-turbo-[0-4]", - ] - ) - - -@pytest.mark.parametrize( - "value,want", - [ - ({"a": 1}, "a=1"), - ({"a": "two"}, "a=two"), - ({"a": [3, 4]}, "a=3,4"), - ({"a": ["five", "six"]}, "a=five,six"), - ({"a": None}, ""), - ({"a": ["seven", None, 8]}, "a=seven,8"), - ({"a": 1, "b": "two"}, "a=1 b=two"), - ({"a": 1, "b": None, "c": "three"}, "a=1 c=three"), - ({"a": 0, "b": None, "c": 0.0, "e": ""}, "a=0 c=0.0"), - ({"a": [0, 0.0, None, "X", "", "Y"]}, "a=0,0.0,X,,Y"), - ]) -def test_dict_to_conf(value: dict, want: str): - assert conf.dict_to_conf(value) == want - - - -@pytest.mark.parametrize( - "cfg,want", - [ - (TstCfg( - install_dir="ukulele", - ), - """LaunchParameters=enable_nss_slurm,use_interactive_step -SlurmctldParameters=cloud_dns,enable_configless,idle_on_node_suspend -SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation -ResumeProgram=ukulele/resume_wrapper.sh -ResumeFailProgram=ukulele/suspend_wrapper.sh -ResumeRate=0 -ResumeTimeout=300 -SuspendProgram=ukulele/suspend_wrapper.sh -SuspendRate=0 -SuspendTimeout=300 -TreeWidth=128 -TopologyPlugin=topology/tree -TopologyParam=SwitchAsNodeRank"""), - (TstCfg( - install_dir="ukulele", - cloud_parameters={ - "no_comma_params": True, - "private_data": None, - "scheduler_parameters": None, - "resume_rate": None, - "resume_timeout": None, - "suspend_rate": None, - "suspend_timeout": None, - "topology_plugin": None, - "topology_param": None, - "tree_width": None, - }, - ), - """SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation -ResumeProgram=ukulele/resume_wrapper.sh -ResumeFailProgram=ukulele/suspend_wrapper.sh -ResumeRate=0 -ResumeTimeout=300 -SuspendProgram=ukulele/suspend_wrapper.sh -SuspendRate=0 -SuspendTimeout=300 -TreeWidth=128 -TopologyPlugin=topology/tree -TopologyParam=SwitchAsNodeRank"""), - (TstCfg( - install_dir="ukulele", - cloud_parameters={ - "no_comma_params": True, - "private_data": [ - "events", - "jobs", - ], - "scheduler_parameters": [ - "bf_busy_nodes", - "bf_continue", - "ignore_prefer_validation", - "nohold_on_prolog_fail", - ], - "resume_rate": 1, - "resume_timeout": 2, - "suspend_rate": 3, - "suspend_timeout": 4, - "topology_plugin": "guess", - "topology_param": "yellow", - "tree_width": 5, - }, - ), - """PrivateData=events,jobs -SchedulerParameters=bf_busy_nodes,bf_continue,ignore_prefer_validation,nohold_on_prolog_fail -ResumeProgram=ukulele/resume_wrapper.sh -ResumeFailProgram=ukulele/suspend_wrapper.sh -ResumeRate=1 -ResumeTimeout=2 -SuspendProgram=ukulele/suspend_wrapper.sh -SuspendRate=3 -SuspendTimeout=4 -TreeWidth=5 -TopologyPlugin=guess -TopologyParam=yellow"""), - (TstCfg( - install_dir="ukulele", - task_prolog_scripts=[Placeholder()], - task_epilog_scripts=[Placeholder()], - ), - """LaunchParameters=enable_nss_slurm,use_interactive_step -SlurmctldParameters=cloud_dns,enable_configless,idle_on_node_suspend -TaskProlog=/slurm/custom_scripts/task_prolog.d/task-prolog -TaskEpilog=/slurm/custom_scripts/task_epilog.d/task-epilog -SchedulerParameters=bf_continue,salloc_wait_nodes,ignore_prefer_validation -ResumeProgram=ukulele/resume_wrapper.sh -ResumeFailProgram=ukulele/suspend_wrapper.sh -ResumeRate=0 -ResumeTimeout=300 -SuspendProgram=ukulele/suspend_wrapper.sh -SuspendRate=0 -SuspendTimeout=300 -TreeWidth=128 -TopologyPlugin=topology/tree -TopologyParam=SwitchAsNodeRank"""), - ]) -def test_conflines(cfg, want): - assert conf.conflines(util.Lookup(cfg)) == want - - cfg.cloud_parameters = addict.Dict(cfg.cloud_parameters) - assert conf.conflines(util.Lookup(cfg)) == want diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py deleted file mode 100644 index 77f1229605..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_resume.py +++ /dev/null @@ -1,175 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional - -import os -import pytest -import unittest.mock -import unittest -import tempfile - -from common import TstCfg, TstNodeset, TstPartition, TstTPU # needed to import util -import util -import resume -from resume import ResumeData, ResumeJobData, BulkChunk, PlacementAndNodes - -def test_get_resume_file_data_no_env(): - with unittest.mock.patch.dict(os.environ, {"SLURM_RESUME_FILE": ""}): - assert resume.get_resume_file_data() is None - - -def test_get_resume_file_data(): - with tempfile.NamedTemporaryFile() as f: - f.write(b"""{ - "jobs": [ - { - "extra": null, - "job_id": 1, - "features": null, - "nodes_alloc": "green-[0-2]", - "nodes_resume": "green-[0-1]", - "oversubscribe": "OK", - "partition": "red", - "reservation": null - } - ], - "all_nodes_resume": "green-[0-1]" -}""") - f.flush() - with ( - unittest.mock.patch.dict(os.environ, {"SLURM_RESUME_FILE": f.name}), - unittest.mock.patch("util.to_hostnames") as mock_to_hostnames, - ): - mock_to_hostnames.return_value = ["green-0", "green-1", "green-2"] - assert resume.get_resume_file_data() == ResumeData(jobs=[ - ResumeJobData( - job_id = 1, - partition="red", - nodes_alloc=["green-0", "green-1", "green-2"], - ) - ]) - mock_to_hostnames.assert_called_once_with("green-[0-2]") - - -@unittest.mock.patch("tpu.TPU.make") -@unittest.mock.patch("resume.create_placements") -def test_group_nodes_bulk(mock_create_placements, mock_tpu): - cfg = TstCfg( - nodeset={ - "n": TstNodeset(nodeset_name="n"), - }, - nodeset_tpu={ - "t": TstNodeset(nodeset_name="t"), - }, - partitions={ - "p1": TstPartition( - partition_name="p1", - enable_job_exclusive=True, - ), - "p2": TstPartition( - partition_name="p2", - partition_nodeset_tpu=["t"], - enable_job_exclusive=True, - ) - } - ) - lkp = util.Lookup(cfg) - - def mock_create_placements_se(nodes, excl_job_id, lkp): - args = (set(nodes), excl_job_id) - if ({'c-n-1', 'c-n-2', 'c-t-8', 'c-t-9'}, None) == args: - return [ - PlacementAndNodes("g0", ["c-n-1", "c-n-2"]), - PlacementAndNodes(None, ['c-t-8', 'c-t-9']), - ] - if ({"c-n-0", "c-n-8"}, 1) == args: - return [ - PlacementAndNodes("g10", ["c-n-0"]), - PlacementAndNodes("g11", ["c-n-8"]), - ] - if ({'c-t-0', 'c-t-1', 'c-t-2', 'c-t-3', 'c-t-4', 'c-t-5'}, 2) == args: - return [ - PlacementAndNodes(None, ['c-t-0', 'c-t-1', 'c-t-2', 'c-t-3', 'c-t-4', 'c-t-5']) - ] - raise AssertionError(f"unexpected invocation: '{args}'") - mock_create_placements.side_effect = mock_create_placements_se - - def mock_tpu_se(ns: str, lkp) -> TstTPU: - if ns == "t": - return TstTPU(vmcount=2) - raise AssertionError(f"unexpected invocation: '{ns}'") - mock_tpu.side_effect = mock_tpu_se - - got = resume.group_nodes_bulk( - ["c-n-0", "c-n-1", "c-n-2", "c-t-0", "c-t-1", "c-t-2", "c-t-3", "c-t-8", "c-t-9"], - ResumeData(jobs=[ - ResumeJobData(job_id=1, partition="p1", nodes_alloc=["c-n-0", "c-n-8"]), - ResumeJobData(job_id=2, partition="p2", nodes_alloc=["c-t-0", "c-t-1", "c-t-2", "c-t-3", "c-t-4", "c-t-5"]), - ]), lkp) - mock_create_placements.assert_called() - assert got == { - "c-n:jobNone:g0:0": BulkChunk( - nodes=["c-n-1", "c-n-2"], prefix="c-n", chunk_idx=0, excl_job_id=None, placement_group="g0"), - "c-n:job1:g10:0": BulkChunk( - nodes=["c-n-0"], prefix="c-n", chunk_idx=0, excl_job_id=1, placement_group="g10"), - "c-t:0": BulkChunk( - nodes=["c-t-8", "c-t-9"], prefix="c-t", chunk_idx=0, excl_job_id=None, placement_group=None), - "c-t:job2:0": BulkChunk( - nodes=["c-t-0", "c-t-1"], prefix="c-t", chunk_idx=0, excl_job_id=2, placement_group=None), - "c-t:job2:1": BulkChunk( - nodes=["c-t-2", "c-t-3"], prefix="c-t", chunk_idx=1, excl_job_id=2, placement_group=None), - } - - -@pytest.mark.parametrize( - "nodes,excl_job_id,expected", - [ - ( # TPU - no placements - ["c-t-0", "c-t-2"], 4, [PlacementAndNodes(None, ["c-t-0", "c-t-2"])] - ), - ( # disabled placements - no placemens - ["c-x-0", "c-x-2"], 4, [PlacementAndNodes(None, ["c-x-0", "c-x-2"])] - ), - ( # excl_job - ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"], 4, [ - PlacementAndNodes("c-slurmgcp-managed-n-4-0", ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"]) - ] - ), - ( # no excl_job - ["c-n-0", "c-n-uno", "c-n-2", "c-n-2011"], None, [ - PlacementAndNodes("c-slurmgcp-managed-n-0-0", ["c-n-0", "c-n-2"]), - PlacementAndNodes('c-slurmgcp-managed-n-0-1', ['c-n-2011']), - PlacementAndNodes(None, ["c-n-uno"]), - ] - ), - ], -) -def test_allocate_nodes_to_placements(nodes: list[str], excl_job_id: Optional[int], expected: list[PlacementAndNodes]): - cfg = TstCfg( - slurm_cluster_name="c", - nodeset={ - "n": TstNodeset(nodeset_name="n", enable_placement=True), - "x": TstNodeset(nodeset_name="x", enable_placement=False) - }, - nodeset_tpu={ - "t": TstNodeset(nodeset_name="t") - }) - lkp = util.Lookup(cfg) - - with unittest.mock.patch("resume.valid_placement_node") as mock_valid_placement_node: - mock_valid_placement_node.return_value = True - lkp.template_info = unittest.mock.Mock(return_value=unittest.mock.Mock(machine_type=unittest.mock.Mock(family="n1"))) - - assert resume._allocate_nodes_to_placements(nodes, excl_job_id, lkp) == expected diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py deleted file mode 100644 index df9f3a0137..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_topology.py +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -import json -import mock -from pytest_unordered import unordered -from common import TstCfg, TstNodeset, TstTPU, tstInstance -import sort_nodes - -import util -import conf -import tempfile - -PRELUDE = """ -# Warning: -# This file is managed by a script. Manual modifications will be overwritten. - -""" - -def test_gen_topology_conf_empty(): - out_dir = tempfile.mkdtemp() - cfg = TstCfg(output_dir=out_dir) - conf.gen_topology_conf(util.Lookup(cfg)) - assert open(out_dir + "/cloud_topology.conf").read() == PRELUDE + "\n" - - -@mock.patch("tpu.TPU.make") -def test_gen_topology_conf(tpu_mock): - output_dir = tempfile.mkdtemp() - cfg = TstCfg( - nodeset_tpu={ - "a": TstNodeset("bold", node_count_static=4, node_count_dynamic_max=5), - "b": TstNodeset("slim", node_count_dynamic_max=3), - }, - nodeset={ - "c": TstNodeset("green", node_count_static=2, node_count_dynamic_max=3), - "d": TstNodeset("blue", node_count_static=7), - "e": TstNodeset("pink", node_count_dynamic_max=4), - }, - output_dir=output_dir, - ) - - def tpu_se(ns: str, lkp) -> TstTPU: - if ns == "bold": - return TstTPU(vmcount=3) - if ns == "slim": - return TstTPU(vmcount=1) - raise AssertionError(f"unexpected TPU name: '{ns}'") - - tpu_mock.side_effect = tpu_se - - lkp = util.Lookup(cfg) - lkp.instances = lambda: { n.name: n for n in [ # type: ignore[assignment] - # nodeset blue - tstInstance("m22-blue-0"), # no physicalHost - tstInstance("m22-blue-0", physical_host="/a/a/a"), - tstInstance("m22-blue-1", physical_host="/a/a/b"), - tstInstance("m22-blue-2", physical_host="/a/b/a"), - tstInstance("m22-blue-3", physical_host="/b/a/a"), - # nodeset green - tstInstance("m22-green-3", physical_host="/a/a/c"), - ]} - - uncompressed = conf.gen_topology(lkp) - want_uncompressed = [ - #NOTE: the switch names are not unique, it's not valid content for topology.conf - # The uniquefication and compression of names are done in the compress() method - "SwitchName=slurm-root Switches=a,b,ns_blue,ns_green,ns_pink", - # "physical" topology - 'SwitchName=a Switches=a,b', - 'SwitchName=a Nodes=m22-blue-[0-1],m22-green-3', - 'SwitchName=b Nodes=m22-blue-2', - 'SwitchName=b Switches=a', - 'SwitchName=a Nodes=m22-blue-3', - # topology "by nodeset" - "SwitchName=ns_blue Nodes=m22-blue-[4-6]", - "SwitchName=ns_green Nodes=m22-green-[0-2,4]", - "SwitchName=ns_pink Nodes=m22-pink-[0-3]", - # TPU topology - "SwitchName=tpu-root Switches=ns_bold,ns_slim", - "SwitchName=ns_bold Switches=bold-[0-3]", - "SwitchName=bold-0 Nodes=m22-bold-[0-2]", - "SwitchName=bold-1 Nodes=m22-bold-3", - "SwitchName=bold-2 Nodes=m22-bold-[4-6]", - "SwitchName=bold-3 Nodes=m22-bold-[7-8]", - "SwitchName=ns_slim Nodes=m22-slim-[0-2]"] - assert list(uncompressed.render_conf_lines()) == want_uncompressed - - compressed = uncompressed.compress() - want_compressed = [ - "SwitchName=s0 Switches=s0_[0-4]", # root - # "physical" topology - 'SwitchName=s0_0 Switches=s0_0_[0-1]', # /a - 'SwitchName=s0_0_0 Nodes=m22-blue-[0-1],m22-green-3', # /a/a - 'SwitchName=s0_0_1 Nodes=m22-blue-2', # /a/b - 'SwitchName=s0_1 Switches=s0_1_0', # /b - 'SwitchName=s0_1_0 Nodes=m22-blue-3', # /b/a - # topology "by nodeset" - "SwitchName=s0_2 Nodes=m22-blue-[4-6]", - "SwitchName=s0_3 Nodes=m22-green-[0-2,4]", - "SwitchName=s0_4 Nodes=m22-pink-[0-3]", - # TPU topology - "SwitchName=s1 Switches=s1_[0-1]", - "SwitchName=s1_0 Switches=s1_0_[0-3]", - "SwitchName=s1_0_0 Nodes=m22-bold-[0-2]", - "SwitchName=s1_0_1 Nodes=m22-bold-3", - "SwitchName=s1_0_2 Nodes=m22-bold-[4-6]", - "SwitchName=s1_0_3 Nodes=m22-bold-[7-8]", - "SwitchName=s1_1 Nodes=m22-slim-[0-2]"] - assert list(compressed.render_conf_lines()) == want_compressed - - upd, summary = conf.gen_topology_conf(lkp) - assert upd == True - want_written = PRELUDE + "\n".join(want_compressed) + "\n\n" - assert open(output_dir + "/cloud_topology.conf").read() == want_written - - summary.dump(lkp) - summary_got = json.loads(open(output_dir + "/cloud_topology.summary.json").read()) - - assert summary_got == { - "down_nodes": unordered( - [f"m22-blue-{i}" for i in (4,5,6)] + - [f"m22-green-{i}" for i in (0,1,2,4)] + - [f"m22-pink-{i}" for i in range(4)]), - "tpu_nodes": unordered( - [f"m22-bold-{i}" for i in range(9)] + - [f"m22-slim-{i}" for i in range(3)]), - 'physical_host': { - 'm22-blue-0': '/a/a/a', - 'm22-blue-1': '/a/a/b', - 'm22-blue-2': '/a/b/a', - 'm22-blue-3': '/b/a/a', - 'm22-green-3': '/a/a/c'}, - } - - - -def test_gen_topology_conf_update(): - cfg = TstCfg( - nodeset={ - "c": TstNodeset("green", node_count_static=2), - }, - output_dir=tempfile.mkdtemp(), - ) - lkp = util.Lookup(cfg) - lkp.instances = lambda: { # type: ignore[assignment] - # no instances - } - - # initial generation - reconfigure - upd, sum = conf.gen_topology_conf(lkp) - assert upd == True - sum.dump(lkp) - - # add node: node_count_static 2 -> 3 - reconfigure - lkp.cfg.nodeset["c"].node_count_static = 3 - upd, sum = conf.gen_topology_conf(lkp) - assert upd == True - sum.dump(lkp) - - # remove node: node_count_static 3 -> 2 - no reconfigure - lkp.cfg.nodeset["c"].node_count_static = 2 - upd, sum = conf.gen_topology_conf(lkp) - assert upd == False - # don't dump - - # set empty physicalHost - no reconfigure - lkp.instances = lambda: { # type: ignore[assignment] - n.name: n for n in [tstInstance("m22-green-0", physical_host="")]} - upd, sum = conf.gen_topology_conf(lkp) - assert upd == False - # don't dump - - # set physicalHost - reconfigure - lkp.instances = lambda: { # type: ignore[assignment] - n.name: n for n in [tstInstance("m22-green-0", physical_host="/a/b/c")]} - upd, sum = conf.gen_topology_conf(lkp) - assert upd == True - sum.dump(lkp) - - # change physicalHost - reconfigure - lkp.instances = lambda: { # type: ignore[assignment] - n.name: n for n in [tstInstance("m22-green-0", physical_host="/a/b/z")]} - upd, sum = conf.gen_topology_conf(lkp) - assert upd == True - sum.dump(lkp) - - # shut down node - no reconfigure - lkp.instances = lambda: {} # type: ignore[assignment] - upd, sum = conf.gen_topology_conf(lkp) - assert upd == False - # don't dump - - -@pytest.mark.parametrize( - "paths,expected", - [ - (["z/n-0", "z/n-1", "z/n-2", "z/n-3", "z/n-4", "z/n-10"], ['n-0', 'n-1', 'n-2', 'n-3', 'n-4', 'n-10']), - (["y/n-0", "z/n-1", "x/n-2", "x/n-3", "y/n-4", "g/n-10"], ['n-0', 'n-4', 'n-1', 'n-2', 'n-3', 'n-10']), - ]) -def test_sort_nodes_order(paths: list[str], expected: list[str]) -> None: - paths_expanded = [l.split("/") for l in paths] - assert sort_nodes.order(paths_expanded) == expected diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py deleted file mode 100644 index d6b42f82e5..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tests/test_util.py +++ /dev/null @@ -1,586 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional, Type - -import pytest -from mock import Mock -from datetime import datetime, timezone, timedelta -import unittest - -from common import TstNodeset, TstCfg # needed to import util -import util -from util import NodeState, MachineType, AcceleratorInfo, UpcomingMaintenance, InstanceResourceStatus, FutureReservation, ReservationDetails -from google.api_core.client_options import ClientOptions # noqa: E402 - -# Note: need to install pytest-mock - -@pytest.mark.parametrize( - "name,expected", - [ - ( - "az-buka-23", - { - "cluster": "az", - "nodeset": "buka", - "node": "23", - "prefix": "az-buka", - "range": None, - "suffix": "23", - }, - ), - ( - "az-buka-xyzf", - { - "cluster": "az", - "nodeset": "buka", - "node": "xyzf", - "prefix": "az-buka", - "range": None, - "suffix": "xyzf", - }, - ), - ( - "az-buka-[2-3]", - { - "cluster": "az", - "nodeset": "buka", - "node": "[2-3]", - "prefix": "az-buka", - "range": "[2-3]", - "suffix": None, - }, - ), - ], -) -def test_node_desc(name, expected): - assert util.lookup()._node_desc(name) == expected - - -@pytest.mark.parametrize( - "name,expected", - [ - ("az-buka-23", 23), - ("az-buka-0", 0), - ("az-buka", Exception), - ("az-buka-xyzf", ValueError), - ("az-buka-[2-3]", ValueError), - ], -) -def test_node_index(name, expected): - if type(expected) is type and issubclass(expected, Exception): - with pytest.raises(expected): - util.lookup().node_index(name) - else: - assert util.lookup().node_index(name) == expected - - -@pytest.mark.parametrize( - "name", - [ - "az-buka", - ], -) -def test_node_desc_fail(name): - with pytest.raises(Exception): - util.lookup()._node_desc(name) - - -@pytest.mark.parametrize( - "names,expected", - [ - ("pedro,pedro-1,pedro-2,pedro-01,pedro-02", "pedro,pedro-[1-2,01-02]"), - ("pedro,,pedro-1,,pedro-2", "pedro,pedro-[1-2]"), - ("pedro-8,pedro-9,pedro-10,pedro-11", "pedro-[8-9,10-11]"), - ("pedro-08,pedro-09,pedro-10,pedro-11", "pedro-[08-11]"), - ("pedro-08,pedro-09,pedro-8,pedro-9", "pedro-[8-9,08-09]"), - ("pedro-10,pedro-08,pedro-09,pedro-8,pedro-9", "pedro-[8-9,08-10]"), - ("pedro-8,pedro-9,juan-10,juan-11", "juan-[10-11],pedro-[8-9]"), - ("az,buki,vedi", "az,buki,vedi"), - ("a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12", "a[0-9,10-12]"), - ("a0,a2,a4,a6,a7,a8,a11,a12", "a[0,2,4,6-8,11-12]"), - ("seas7-0,seas7-1", "seas7-[0-1]"), - ], -) -def test_to_hostlist(names, expected): - assert util.to_hostlist(names.split(",")) == expected - - -@pytest.mark.parametrize( - "api,ep_ver,expected", - [ - ( - util.ApiEndpoint.BQ, - "v1", - ClientOptions(api_endpoint="https://bq.googleapis.com/v1/"), - ), - ( - util.ApiEndpoint.COMPUTE, - "staging_v1", - ClientOptions(api_endpoint="https://compute.googleapis.com/staging_v1/"), - ), - ( - util.ApiEndpoint.SECRET, - "v1", - ClientOptions(api_endpoint="https://secret_manager.googleapis.com/v1/"), - ), - ( - util.ApiEndpoint.STORAGE, - "beta", - ClientOptions(api_endpoint="https://storage.googleapis.com/beta/"), - ), - ( - util.ApiEndpoint.TPU, - "alpha", - ClientOptions(api_endpoint="https://tpu.googleapis.com/alpha/"), - ), - ], -) -def test_create_client_options( - api: util.ApiEndpoint, ep_ver: str, expected: ClientOptions, mocker -): - ud_mock = mocker.patch("util.universe_domain") - ep_mock = mocker.patch("util.endpoint_version") - ud_mock.return_value = "googleapis.com" - ep_mock.return_value = ep_ver - assert util.create_client_options(api).__repr__() == expected.__repr__() - - - -@pytest.mark.parametrize( - "nodeset,err", - [ - (TstNodeset(reservation_name="projects/x/reservations/y"), AssertionError), # no zones - (TstNodeset( - reservation_name="projects/x/reservations/y", - zone_policy_allow=["eine", "zwei"]), AssertionError), # multiples zones - (TstNodeset( - reservation_name="robin", - zone_policy_allow=["eine"]), ValueError), # invalid name - (TstNodeset( - reservation_name="projects/reservations/y", - zone_policy_allow=["eine"]), ValueError), # invalid name - (TstNodeset( - reservation_name="projects/x/zones/z/reservations/y", - zone_policy_allow=["eine"]), ValueError), # invalid name - ] -) -def test_nodeset_reservation_err(nodeset, err): - lkp = util.Lookup(TstCfg()) - lkp._get_reservation = Mock() - with pytest.raises(err): - lkp.nodeset_reservation(nodeset) - lkp._get_reservation.assert_not_called() # type: ignore - -@pytest.mark.parametrize( - "nodeset,policies,expected", - [ - (TstNodeset(), [], None), # no reservation - (TstNodeset( - reservation_name="projects/bobin/reservations/robin", - zone_policy_allow=["eine"]), - [], - util.ReservationDetails( - project="bobin", - zone="eine", - name="robin", - policies=[], - deployment_type=None, - reservation_mode=None, - bulk_insert_name="projects/bobin/reservations/robin")), - (TstNodeset( - reservation_name="projects/bobin/reservations/robin", - zone_policy_allow=["eine"]), - ["seven/wanders", "five/red/apples", "yum"], - util.ReservationDetails( - project="bobin", - zone="eine", - name="robin", - policies=["wanders", "apples", "yum"], - deployment_type=None, - reservation_mode=None, - bulk_insert_name="projects/bobin/reservations/robin")), - (TstNodeset( - reservation_name="projects/bobin/reservations/robin/snek/cheese-brie-6", - zone_policy_allow=["eine"]), - [], - util.ReservationDetails( - project="bobin", - zone="eine", - name="robin", - policies=[], - deployment_type=None, - reservation_mode=None, - bulk_insert_name="projects/bobin/reservations/robin/snek/cheese-brie-6")), - - ]) - -def test_nodeset_reservation_ok(nodeset, policies, expected): - lkp = util.Lookup(TstCfg()) - lkp._get_reservation = Mock() - - if not expected: - assert lkp.nodeset_reservation(nodeset) is None - lkp._get_reservation.assert_not_called() # type: ignore - return - - lkp._get_reservation.return_value = { # type: ignore - "resourcePolicies": {i: p for i, p in enumerate(policies)}, - } - assert lkp.nodeset_reservation(nodeset) == expected - lkp._get_reservation.assert_called_once_with(expected.project, expected.zone, expected.name) # type: ignore - -@pytest.mark.parametrize( - "job_info,expected_job", - [ - ( - """JobId=123 - TimeLimit=02:00:00 - JobName=myjob - JobState=PENDING - ReqNodeList=node-[1-10]""", - util.Job( - id=123, - duration=timedelta(days=0, hours=2, minutes=0, seconds=0), - name="myjob", - job_state="PENDING", - required_nodes="node-[1-10]" - ), - ), - ( - """JobId=456 - JobName=anotherjob - JobState=PENDING - ReqNodeList=node-group1""", - util.Job( - id=456, - duration=None, - name="anotherjob", - job_state="PENDING", - required_nodes="node-group1" - ), - ), - ( - """JobId=789 - TimeLimit=00:30:00 - JobState=COMPLETED""", - util.Job( - id=789, - duration=timedelta(minutes=30), - name=None, - job_state="COMPLETED", - required_nodes=None - ), - ), - ( - """JobId=101112 - TimeLimit=1-00:30:00 - JobState=COMPLETED, - ReqNodeList=node-[1-10],grob-pop-[2,1,44-77]""", - util.Job( - id=101112, - duration=timedelta(days=1, hours=0, minutes=30, seconds=0), - name=None, - job_state="COMPLETED", - required_nodes="node-[1-10],grob-pop-[2,1,44-77]" - ), - ), - ( - """JobId=131415 - TimeLimit=1-00:30:00 - JobName=mynode-1_maintenance - JobState=COMPLETED, - ReqNodeList=node-[1-10],grob-pop-[2,1,44-77]""", - util.Job( - id=131415, - duration=timedelta(days=1, hours=0, minutes=30, seconds=0), - name="mynode-1_maintenance", - job_state="COMPLETED", - required_nodes="node-[1-10],grob-pop-[2,1,44-77]" - ), - ), - ], -) -def test_parse_job_info(job_info, expected_job): - lkp = util.Lookup(TstCfg()) - assert lkp._parse_job_info(job_info) == expected_job - - - -@pytest.mark.parametrize( - "node,state,want", - [ - ("c-n-2", NodeState("DOWN", frozenset([])), NodeState("DOWN", frozenset([]))), # happy scenario - ("c-d-vodoo", None, None), # dynamic nodeset - ("c-x-44", None, None), # unknown(removed) nodeset - ("c-n-7", None, None), # Out of bounds: c-n-[0-4] - downsized nodeset - ("c-t-7", None, None), # Out of bounds: c-t-[0-4] - downsized nodeset TPU - ("c-n-2", None, RuntimeError), # something is wrong - ("c-t-2", None, RuntimeError), # something is wrong, but TPU - - # Check boundaries match [0-5) - ("c-n-5", None, None), # out of boundaries - ("c-n-4", None, RuntimeError), # within boundaries - ]) -def test_node_state(node: str, state: Optional[NodeState], want: NodeState | None | Type[Exception]): - cfg = TstCfg( - slurm_cluster_name="c", - nodeset={ - "n": TstNodeset(node_count_static=2, node_count_dynamic_max=3)}, - nodeset_tpu={ - "t": TstNodeset(node_count_static=2, node_count_dynamic_max=3)}, - nodeset_dyn={ - "d": TstNodeset()}, - ) - lkp = util.Lookup(cfg) - lkp.slurm_nodes = lambda: {node: state} if state else {} # type: ignore[assignment] - # ... see https://github.com/python/typeshed/issues/6347 - - if type(want) is type and issubclass(want, Exception): - with pytest.raises(want): - lkp.node_state(node) - else: - assert lkp.node_state(node) == want - - - -@pytest.mark.parametrize( - "jo,want", - [ - ({ - "accelerators": [ { "guestAcceleratorCount": 1, "guestAcceleratorType": "nvidia-tesla-a100" } ], - "creationTimestamp": "1969-12-31T16:00:00.000-08:00", - "description": "Accelerator Optimized: 1 NVIDIA Tesla A100 GPU, 12 vCPUs, 85GB RAM", - "guestCpus": 12, - "id": "1000012", - "imageSpaceGb": 0, - "isSharedCpu": False, - "kind": "compute#machineType", - "maximumPersistentDisks": 128, - "maximumPersistentDisksSizeGb": "263168", - "memoryMb": 87040, - "name": "a2-highgpu-1g", - "selfLink": "https://www.googleapis.com/compute/v1/projects/io-playground/zones/us-central1-a/machineTypes/a2-highgpu-1g", - "zone": "us-central1-a" - }, MachineType( - name="a2-highgpu-1g", - guest_cpus=12, - memory_mb=87040, - accelerators=[ - AcceleratorInfo(type="nvidia-tesla-a100", count=1) - ] - )), - ({ - "architecture": "X86_64", - "creationTimestamp": "1969-12-31T16:00:00.000-08:00", - "description": "8 vCPUs, 32 GB RAM", - "guestCpus": 8, - "id": "1210008", - "imageSpaceGb": 0, - "isSharedCpu": False, - "kind": "compute#machineType", - "maximumPersistentDisks": 128, - "maximumPersistentDisksSizeGb": "263168", - "memoryMb": 32768, - "name": "t2d-standard-8", - "selfLink": "https://www.googleapis.com/compute/v1/projects/io-playground/zones/europe-north2-b/machineTypes/t2d-standard-8", - "zone": "europe-north2-b" - }, MachineType( - name="t2d-standard-8", - guest_cpus=8, - memory_mb=32768, - accelerators=[] - )), - ]) -def test_MachineType_from_json(jo: dict, want: MachineType): - assert MachineType.from_json(jo) == want - -UTC, PST = timezone.utc, timezone(timedelta(hours=-8)) - -@pytest.mark.parametrize( - "got,want", - [ - # from instance.creationTimestamp: - ("2024-11-30T12:47:51.676-08:00", datetime(2024, 11, 30, 12, 47, 51, 676000, tzinfo=PST)), - # from futureReservation.creationTimestamp - ("2024-11-05T15:23:33.702-08:00", datetime(2024, 11, 5, 15, 23, 33, 702000, tzinfo=PST)), - # from futureReservation.timeWindow.endTime - ("2025-01-15T00:00:00Z", datetime(2025, 1, 15, 0, 0, tzinfo=UTC)), - # fallback to UTC if no tz is specified - ("2025-01-15T00:00:00", datetime(2025, 1, 15, 0, 0, tzinfo=UTC)), - ]) -def test_parse_gcp_timestamp(got: str, want: datetime): - assert util.parse_gcp_timestamp(got) == want - - -@pytest.mark.parametrize( - "got,want", - [ - (None, None), - (dict( - windowStartTime="2025-01-15T00:00:00Z", - somethingToIgnore="past failures", - ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), - (dict( - startTimeWindow=dict( - earliest="2025-01-15T00:00:00Z"), - somethingToIgnore="past failures", - ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), - (dict( - windowStartTime="2025-01-15T00:00:00Z", - startTimeWindow=dict( - earliest="2025-01-25T00:00:00Z"), # ignored - somethingToIgnore="past failures", - ), UpcomingMaintenance(window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC))), - ]) -def tests_parse_UpcomingMaintenance_OK(got: dict, want: Optional[UpcomingMaintenance]): - assert UpcomingMaintenance.from_json(got) == want - - -@pytest.mark.parametrize( - "got", - [ - {}, - dict( - windowStartTime=dict( - earliest="2025-01-15T00:00:00Z")), - ]) -def tests_parse_UpcomingMaintenance_FAIL(got: dict): - with pytest.raises(ValueError): - UpcomingMaintenance.from_json(got) - - -@pytest.mark.parametrize( - "got,want", - [ - (None, InstanceResourceStatus( - physical_host=None, - upcoming_maintenance=None)), - ({}, InstanceResourceStatus( - physical_host=None, - upcoming_maintenance=None)), - (dict( - physicalHost="/aaa/bbb/ccc"), - InstanceResourceStatus( - physical_host="/aaa/bbb/ccc", - upcoming_maintenance=None)), - (dict( # invalid upcomingMaintenance field to be ignored - physicalHost="/aaa/bbb/ccc", - upcomingMaintenance="maintenance is upon us"), - InstanceResourceStatus( - physical_host="/aaa/bbb/ccc", - upcoming_maintenance=None)), - (dict( - physicalHost="/aaa/bbb/ccc", - upcomingMaintenance=dict(windowStartTime="2025-01-15T00:00:00Z")), - InstanceResourceStatus( - physical_host="/aaa/bbb/ccc", - upcoming_maintenance=UpcomingMaintenance( - window_start_time=datetime(2025, 1, 15, 0, 0, tzinfo=UTC)))), - ]) -def test_parse_InstanceResourceStatus(got: dict, want: Optional[InstanceResourceStatus]): - assert InstanceResourceStatus.from_json(got) == want - - -def test_future_reservation_none(): - lkp = util.Lookup(TstCfg()) - assert lkp.future_reservation(TstNodeset()) == None - - -def test_future_reservation_declined(): - lkp = util.Lookup(TstCfg()) - lkp._get_future_reservation = Mock(return_value=dict( - timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-03T23:30:00Z" }, - status = {"procurementStatus": "DECLINED"}, - reservationMode = "CALENDAR", - specificReservationRequired = True, - )) - - assert lkp.future_reservation( - TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( - project='manhattan', - zone='danger', - name='zebra', - specific=True, - start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), - end_time=datetime(2025, 2, 3, 23, 30, tzinfo=timezone.utc), - reservation_mode="CALENDAR", - active_reservation=None) - lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") - -@unittest.mock.patch('util.now', return_value=datetime(2025, 2, 13, 0, 0, tzinfo=timezone.utc)) -def test_future_reservation_active(_): - lkp = util.Lookup(TstCfg()) - lkp._get_future_reservation = Mock(return_value=dict( - timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-21T23:30:00Z" }, - status = { - "procurementStatus": "FULFILLED", - "autoCreatedReservations": [ - "https://www.googleapis.com/compute/alpha/projects/manhattan/zones/danger/reservations/melon" - ], - }, - specificReservationRequired = True, - )) - lkp._get_reservation = Mock(return_value=dict()) - - assert lkp.future_reservation( - TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( - project='manhattan', - zone='danger', - name='zebra', - specific=True, - start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), - end_time=datetime(2025, 2, 21, 23, 30, tzinfo=timezone.utc), - reservation_mode=None, - active_reservation=ReservationDetails( - project='manhattan', - zone='danger', - name='melon', - policies=[], - reservation_mode=None, - bulk_insert_name="projects/manhattan/reservations/melon", - deployment_type=None)) - - lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") - lkp._get_reservation.assert_called_once_with("manhattan", "danger", "melon") - -@unittest.mock.patch('util.now', return_value=datetime(2025, 2, 28, 0, 0, tzinfo=timezone.utc)) -def test_future_reservation_inactive(_): - lkp = util.Lookup(TstCfg()) - lkp._get_future_reservation = Mock(return_value=dict( - timeWindow = { "startTime": "2025-01-27T23:30:00Z", "endTime": "2025-02-21T23:30:00Z" }, - status = { - "procurementStatus": "FULFILLED", - "autoCreatedReservations": [ - "https://www.googleapis.com/compute/alpha/projects/manhattan/zones/danger/reservations/melon" - ], - }, - reservationMode = "DEFAULT", - specificReservationRequired = True, - )) - lkp._get_reservation = Mock() - - assert lkp.future_reservation( - TstNodeset(future_reservation="projects/manhattan/zones/danger/futureReservations/zebra")) == FutureReservation( - project='manhattan', - zone='danger', - name='zebra', - specific=True, - start_time=datetime(2025, 1, 27, 23, 30, tzinfo=timezone.utc), - end_time=datetime(2025, 2, 21, 23, 30, tzinfo=timezone.utc), - reservation_mode="DEFAULT", - active_reservation=None) - - lkp._get_future_reservation.assert_called_once_with("manhattan", "danger", "zebra") - lkp._get_reservation.assert_not_called() diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test deleted file mode 100644 index 46330331d4..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/gpu-test +++ /dev/null @@ -1,129 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -LOG_FILE="/var/log/slurm/chs_health_check.log" -TMP_DCGM_OUT="/tmp/dcgm.out" -TMP_ECC_ERRORS_OUT="/tmp/ecc_errors.out" - -log_step() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" -} - -# Fail gracefully if nvidia-smi or dcgmi doesn't exist -if ! type -P nvidia-smi 1>/dev/null; then - log_step "nvidia-smi not found - this script requires nvidia-smi to function" - exit 0 -fi - -if ! type -P dcgmi 1>/dev/null; then - log_step "dcgmi not found - this script requires dcgmi to function" - exit 0 -fi - -if ! type -P nv-hostengine 1>/dev/null; then - log_step "nv-hostengine not found - this script requires nv-hostengine to function" - exit 0 -fi - -################################################### -# Disable running health checks -################################################### -# Check if the environment variable '$SLURM_JOB_EXTRA' is set and contains the -# substring 'healthchecks_prolog=off' -if [[ -n "$SLURM_JOB_EXTRA" ]]; then - log_step "Environment variable SLURM_JOB_EXTRA is set. Checking if it contains healthchecks_prolog=off." - # Check if the value of the variable matches the string "healthchecks_prolog=off" - if [[ "$SLURM_JOB_EXTRA" == *"healthchecks_prolog=off"* ]]; then - log_step "Environment variable SLURM_JOB_EXTRA matches substring healthchecks_prolog=off. Skipping health checks." - exit 0 - else - log_step "Environment variable SLURM_JOB_EXTRA does NOT match substring healthchecks_prolog=off. Attempting to run health checks." - fi -else - log_step "Environment variable SLURM_JOB_EXTRA is NOT set. Attempting to run health checks." -fi - -# Exit if GPU isn't H/B 100/200 -GPU_MODEL=$(nvidia-smi --query-gpu=name --format=csv,noheader) -if ! [[ "$GPU_MODEL" =~ [BH][1-2]00 ]]; then - log_step "No Supported GPU detected" - exit 0 -fi - -NUMGPUS=$(nvidia-smi -L | wc -l) - -# Check that all GPUs are healthy via DCGM and check for ECC errors -if [ $NUMGPUS -gt 0 ]; then - log_step "Execute DCGM health check, ECC error check, and NVLink error check for GPUs" - GPULIST=$(nvidia-smi --query-gpu=index --format=csv,noheader | tr '\n' ',' | sed 's/,$//') - rm -f $TMP_DCGM_OUT - rm -f $TMP_ECC_ERRORS_OUT - - # Run DCGM checks - START_HOSTENGINE=false - if ! pidof nv-hostengine > /dev/null; then - log_step "Starting nv-hostengine..." - nv-hostengine >> "$LOG_FILE" 2>&1 - sleep 1 # Give it a moment to start up - START_HOSTENGINE=true - fi - GROUPID=$(dcgmi group -c gpuinfo | awk '{print $NF}' | tr -d ' ') - dcgmi group -g $GROUPID -a $GPULIST >> "$LOG_FILE" 2>&1 - dcgmi diag -g $GROUPID -r 1 > "$TMP_DCGM_OUT" 2>&1 - cat "$TMP_DCGM_OUT" >> "$LOG_FILE" - dcgmi group -d $GROUPID >> "$LOG_FILE" 2>&1 - - # Terminate the host engine if it was manually started - if [ "$START_HOSTENGINE" = true ]; then - log_step "Terminating nv-hostengine..." - nv-hostengine -t >> "$LOG_FILE" 2>&1 - fi - - # Check for DCGM failures - DCGM_FAILED=0 - if grep -i fail "$TMP_DCGM_OUT" > /dev/null; then - DCGM_FAILED=1 - fi - - # Check for ECC errors - nvidia-smi --query-gpu=ecc.errors.uncorrected.volatile.total --format=csv,noheader > "$TMP_ECC_ERRORS_OUT" - cat "$TMP_ECC_ERRORS_OUT" >> "$LOG_FILE" - ECC_ERRORS=$(awk -F', ' '{sum += $2} END {print sum}' "$TMP_ECC_ERRORS_OUT") - log_step "ECC Errors: $ECC_ERRORS" - - # Check for NVLink errors - NVLINK_ERRORS=$(nvidia-smi nvlink -sc 0bz -i 0 2>/dev/null | grep -i "Error Count" | awk '{sum += $3} END {print sum}') - # Set to 0 if empty/null - NVLINK_ERRORS=${NVLINK_ERRORS:-0} - log_step "NVLink Errors: $NVLINK_ERRORS" - - if [ $DCGM_FAILED -eq 1 ] || \ - [ $ECC_ERRORS -gt 0 ] || \ - [ $NVLINK_ERRORS -gt 0 ]; then - REASON="GPU issues detected: " - if [ $DCGM_FAILED -eq 1 ]; then - REASON+="DCGM test failed, " - fi - if [ $ECC_ERRORS -gt 0 ]; then - REASON+="ECC errors found ($ECC_ERRORS double-bit errors), " - fi - if [ $NVLINK_ERRORS -gt 0 ]; then - REASON+="NVLink errors detected ($NVLINK_ERRORS errors), " - fi - REASON+="see $LOG_FILE" - log_step "$REASON" - exit 1 - fi -fi diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog deleted file mode 100644 index a22ddea9e5..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-epilog +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Main TaskEpilog Script -# This script executes all *.sh scripts found in /slurm/custom_scripts/task_epilog.d/ -# -# slurm.conf configuration: -# TaskEpilog=/slurm/scripts/tools/task-epilog - -# Directory containing the individual task epilog scripts -EPILOG_D_DIR="/slurm/custom_scripts/task_epilog.d" - -# --- Output Handling for TaskEpilog --- -# The stdout and stderr of this script (and the sub-scripts it calls) -# are typically captured by Slurm and written to the job's output/error file -# or a separate Slurm log, depending on configuration. -# Unlike TaskProlog, stdout is not typically parsed for special commands -# like 'export' or 'print' to affect the (now finished) task's environment. -# -# --- Error Handling --- -# If any script in EPILOG_D_DIR exits with a non-zero status, -# this main script will also exit with a non-zero status. -# Slurm will log this. Depending on Slurm's configuration, -# frequent epilog failures might lead to node issues or alerts. -set -e # Exit immediately if a command exits with a non-zero status. - -# Check if the directory exists -if [[ ! -d "$EPILOG_D_DIR" ]]; then - # Log in task stdout and exit if the directory is missing. This likely indicates a configuration error. - echo "print TaskEpilog Error: Directory '$EPILOG_D_DIR' not found. Check Slurm configuration." - exit 1 -fi - -# Find and execute all *.sh scripts in the directory -# Scripts will be executed in reverse alphabetical order of their filenames. -find "$EPILOG_D_DIR" -maxdepth 1 -type f -name "*.sh" -print0 | sort -rz | while IFS= read -r -d $'\0' script; do - if [[ -x "$script" ]]; then - # Execute the script. Its stdout will be captured by this wrapper. - # Its stderr will also be passed through. - # If a sub-script exits with an error, 'set -e' will cause this wrapper to exit. - "$script" - else - # Log in task stdout a warning if a *.sh file is found but is not executable - echo "print TaskEpilog Warning: Script '$script' is not executable and will be skipped." - fi -done - -# Check if any scripts were found and executed -if [[ $(find "$EPILOG_D_DIR" -maxdepth 1 -type f -name "*.sh" | wc -l) -eq 0 ]]; then - # Log in task stdout if no scripts were found to execute - echo "print TaskEpilog Info: No executable *.sh scripts found in $EPILOG_D_DIR." -fi - -# Exit with 0 if all scripts were successful (or no scripts to run and not treated as error) -exit 0 diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog deleted file mode 100644 index feddb23209..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tools/task-prolog +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Main TaskProlog Script -# This script executes all *.sh scripts found in /slurm/custom_scripts/task_prolog.d/ -# -# slurm.conf configuration: -# TaskProlog=/slurm/scripts/tools/task-prolog - -# Directory containing the individual task prolog scripts -PROLOG_D_DIR="/slurm/custom_scripts/task_prolog.d" - -# --- Output Handling for TaskProlog --- -# Slurm's TaskProlog can interpret specific stdout lines: -# - "export NAME=value" : Sets an environment variable for the task. -# - "unset NAME" : Unsets an environment variable for the task. -# - "print message" : Prints a message to the task's standard output. -# -# This wrapper script will concatenate the stdout of all sub-scripts. -# If sub-scripts need to set/unset environment variables or print messages -# for the task, they should output the appropriate "export", "unset", or "print" -# commands to their own stdout. - -# --- Error Handling --- -# If any script in PROLOG_D_DIR exits with a non-zero status, -# this main script will also exit with a non-zero status. -# This will typically cause the task to fail. -set -e # Exit immediately if a command exits with a non-zero status. - -# Check if the directory exists -if [[ ! -d "$PROLOG_D_DIR" ]]; then - # Log in task stdout and exit if the directory is missing. All jobs will be failed. - echo "print TaskProlog Error: Directory '$PROLOG_D_DIR' not found. Check Slurm configuration." - exit 1 -fi - -# Find and execute all *.sh scripts in the directory -# Scripts will be executed in reverse alphabetical order of their filenames. -find "$PROLOG_D_DIR" -maxdepth 1 -type f -name "*.sh" -print0 | sort -rz | while IFS= read -r -d $'\0' script; do - if [[ -x "$script" ]]; then - # Execute the script. Its stdout will be captured by this wrapper. - # Its stderr will also be passed through. - # If a sub-script exits with an error, 'set -e' will cause this wrapper to exit. - "$script" - else - # Log a warning in task stdout if a *.sh file is found but is not executable - echo "print TaskProlog Warning: Script '$script' is not executable and will be skipped." - fi -done - -# Check if any scripts were found and executed -if [[ $(find "$PROLOG_D_DIR" -maxdepth 1 -type f -name "*.sh" | wc -l) -eq 0 ]]; then - # Log in task stdout if no scripts were found to execute - echo "print TaskProlog Info: No executable *.sh scripts found in $PROLOG_D_DIR." -fi - -# Exit with 0 if all scripts were successful (or no scripts to run and not treated as error) -exit 0 diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py deleted file mode 100644 index 531f0348dc..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/tpu.py +++ /dev/null @@ -1,331 +0,0 @@ -# mypy: ignore-errors -# This implementation of TPU integration is to be deprecated - -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List - -import socket -import logging -from pathlib import Path -import yaml - -import util -from util import create_client_options, ApiEndpoint - -from google.cloud import tpu_v2 as tpu # noqa: E402 -import google.api_core.exceptions as gExceptions # noqa: E402 - -log = logging.getLogger() - -_tpu_cache = {} - -class TPU: - """Class for handling the TPU-vm nodes""" - - State = tpu.types.cloud_tpu.Node.State - TPUS_PER_VM = 4 - __expected_states = { - "create": State.READY, - "start": State.READY, - "stop": State.STOPPED, - } - - __tpu_version_mapping = { - "V2": tpu.AcceleratorConfig().Type.V2, - "V3": tpu.AcceleratorConfig().Type.V3, - "V4": tpu.AcceleratorConfig().Type.V4, - } - - @classmethod - def make(cls, nodeset_name: str, lkp: util.Lookup) -> "TPU": - key = (id(lkp), nodeset_name) - if key not in _tpu_cache: - nodeset = lkp.cfg.nodeset_tpu[nodeset_name] - _tpu_cache[key] = cls(nodeset, lkp) - return _tpu_cache[key] - - - def __init__(self, nodeset: object, lkp: util.Lookup): - self._nodeset = nodeset - self.lkp = lkp - self._parent = f"projects/{lkp.project}/locations/{nodeset.zone}" - co = create_client_options(ApiEndpoint.TPU) - self._client = tpu.TpuClient(client_options=co) - self.data_disks = [] - for data_disk in nodeset.data_disks: - ad = tpu.AttachedDisk() - ad.source_disk = data_disk - ad.mode = tpu.AttachedDisk.DiskMode.DISK_MODE_UNSPECIFIED - self.data_disks.append(ad) - ns_ac = nodeset.accelerator_config - if ns_ac.topology != "" and ns_ac.version != "": - ac = tpu.AcceleratorConfig() - ac.topology = ns_ac.topology - ac.type_ = self.__tpu_version_mapping[ns_ac.version] - self.ac = ac - else: - req = tpu.GetAcceleratorTypeRequest( - name=f"{self._parent}/acceleratorTypes/{nodeset.node_type}" - ) - self.ac = self._client.get_accelerator_type(req).accelerator_configs[0] - self.vmcount = self.__calc_vm_from_topology(self.ac.topology) - - @property - def nodeset(self): - return self._nodeset - - @property - def preserve_tpu(self): - return self._nodeset.preserve_tpu - - @property - def node_type(self): - return self._nodeset.node_type - - @property - def tf_version(self): - return self._nodeset.tf_version - - @property - def enable_public_ip(self): - return self._nodeset.enable_public_ip - - @property - def preemptible(self): - return self._nodeset.preemptible - - @property - def reserved(self): - return self._nodeset.reserved - - @property - def service_account(self): - return self._nodeset.service_account - - @property - def zone(self): - return self._nodeset.zone - - def check_node_type(self): - if self.node_type is None: - return False - try: - request = tpu.GetAcceleratorTypeRequest( - name=f"{self._parent}/acceleratorTypes/{self.node_type}" - ) - return self._client.get_accelerator_type(request=request) is not None - except Exception: - return False - - def check_tf_version(self): - try: - request = tpu.GetRuntimeVersionRequest( - name=f"{self._parent}/runtimeVersions/{self.tf_version}" - ) - return self._client.get_runtime_version(request=request) is not None - except Exception: - return False - - def __calc_vm_from_topology(self, topology): - topo = topology.split("x") - tot = 1 - for num in topo: - tot = tot * int(num) - return tot // self.TPUS_PER_VM - - def __check_resp(self, response, op_name): - des_state = self.__expected_states.get(op_name) - # If the state is not in the table just print the response - if des_state is None: - return False - if response.__class__.__name__ != "Node": # If the response is not a node fail - return False - if response.state == des_state: - return True - return False - - def list_nodes(self): - try: - request = tpu.ListNodesRequest(parent=self._parent) - res = self._client.list_nodes(request=request) - except gExceptions.NotFound: - res = None - return res - - def list_node_names(self): - return [node.name.split("/")[-1] for node in self.list_nodes()] - - def start_node(self, nodename): - request = tpu.StartNodeRequest(name=f"{self._parent}/nodes/{nodename}") - resp = self._client.start_node(request=request).result() - return self.__check_resp(resp, "start") - - def stop_node(self, nodename): - request = tpu.StopNodeRequest(name=f"{self._parent}/nodes/{nodename}") - resp = self._client.stop_node(request=request).result() - return self.__check_resp(resp, "stop") - - def get_node(self, nodename): - try: - request = tpu.GetNodeRequest(name=f"{self._parent}/nodes/{nodename}") - res = self._client.get_node(request=request) - except gExceptions.NotFound: - res = None - return res - - def _register_node(self, nodename, ip_addr): - dns_name = socket.getnameinfo((ip_addr, 0), 0)[0] - util.run( - f"{self.lkp.scontrol} update nodename={nodename} nodeaddr={ip_addr} nodehostname={dns_name}" - ) - - def create_node(self, nodename): - if self.vmcount > 1 and not isinstance(nodename, list): - log.error( - f"Tried to create a {self.vmcount} node TPU on nodeset {self._nodeset.nodeset_name} but only received one nodename {nodename}" - ) - return False - if self.vmcount > 1 and ( - isinstance(nodename, list) and len(nodename) != self.vmcount - ): - log.error( - f"Expected to receive a list of {self.vmcount} nodenames for TPU node creation in nodeset {self._nodeset.nodeset_name}, but received this list {nodename}" - ) - return False - - node = tpu.Node() - node.accelerator_config = self.ac - node.runtime_version = f"tpu-vm-tf-{self.tf_version}" - startup_script = """ - #!/bin/bash - echo "startup script not found > /var/log/startup_error.log" - """ - with open( - Path(self.lkp.cfg.slurm_scripts_dir or util.dirs.scripts) / "startup.sh", "r" - ) as script: - startup_script = script.read() - if isinstance(nodename, list): - node_id = nodename[0] - slurm_names = [] - wid = 0 - for node_wid in nodename: - slurm_names.append(f"WORKER_{wid}:{node_wid}") - wid += 1 - else: - node_id = nodename - slurm_names = [f"WORKER_0:{nodename}"] - node.metadata = { - "slurm_docker_image": self.nodeset.docker_image, - "startup-script": startup_script, - "slurm_instance_role": "compute", - "slurm_cluster_name": self.lkp.cfg.slurm_cluster_name, - "slurm_bucket_path": self.lkp.cfg.bucket_path, - "slurm_names": ";".join(slurm_names), - "universe_domain": util.universe_domain(), - } - node.tags = [self.lkp.cfg.slurm_cluster_name] - if self.nodeset.service_account: - node.service_account.email = self.nodeset.service_account.email - node.service_account.scope = self.nodeset.service_account.scopes - node.scheduling_config.preemptible = self.preemptible - node.scheduling_config.reserved = self.reserved - node.network_config.subnetwork = self.nodeset.subnetwork - node.network_config.enable_external_ips = self.enable_public_ip - if self.data_disks: - node.data_disks = self.data_disks - - request = tpu.CreateNodeRequest(parent=self._parent, node=node, node_id=node_id) - resp = self._client.create_node(request=request).result() - if not self.__check_resp(resp, "create"): - return False - if isinstance(nodename, list): - for node_id, net_endpoint in zip(nodename, resp.network_endpoints): - self._register_node(node_id, net_endpoint.ip_address) - else: - ip_add = resp.network_endpoints[0].ip_address - self._register_node(nodename, ip_add) - return True - - def delete_node(self, nodename): - request = tpu.DeleteNodeRequest(name=f"{self._parent}/nodes/{nodename}") - try: - resp = self._client.delete_node(request=request).result() - if resp: - return self.get_node(nodename=nodename) is None - return False - except gExceptions.NotFound: - # log only error if vmcount is 1 as for other tpu vm count, this could be "phantom" nodes - if self.vmcount == 1: - log.error(f"Tpu single node {nodename} not found") - else: - # for the TPU nodes that consist in more than one vm, only the first node of the TPU a.k.a. the master node will - # exist as real TPU nodes, so the other ones are expected to not be found, check the hostname of the node that has - # not been found, and if it ends in 0, it means that is the master node and it should have been found, and in consequence - # log an error - nodehostname = yaml.safe_load( - util.run(f"{self.lkp.scontrol} --yaml show node {nodename}").stdout.rstrip() - )["nodes"][0]["hostname"] - if nodehostname.split("-")[-1] == "0": - log.error(f"TPU master node {nodename} not found") - else: - log.info(f"Deleted TPU 'phantom' node {nodename}") - # If the node is not found it is tecnichally deleted, so return success. - return True - -def _stop_tpu(node: str) -> None: - lkp = util.lookup() - tpuobj = TPU.make(lkp.node_nodeset_name(node), lkp) - if tpuobj.nodeset.preserve_tpu and tpuobj.vmcount == 1: - log.info(f"stopping node {node}") - if tpuobj.stop_node(node): - return - log.error("Error stopping node {node} will delete instead") - log.info(f"deleting node {node}") - if not tpuobj.delete_node(node): - log.error("Error deleting node {node}") - - -def delete_tpu_instances(instances: List[str]) -> None: - util.execute_with_futures(_stop_tpu, instances) - - -def start_tpu(node: List[str]): - lkp = util.lookup() - tpuobj = TPU.make(lkp.node_nodeset_name(node[0]), lkp) - - if len(node) == 1: - node = node[0] - log.debug( - f"Will create a TPU of type {tpuobj.node_type} tf_version {tpuobj.tf_version} in zone {tpuobj.zone} with name {node}" - ) - tpunode = tpuobj.get_node(node) - if tpunode is None: - if not tpuobj.create_node(nodename=node): - log.error("Error creating tpu node {node}") - else: - if tpuobj.preserve_tpu: - if not tpuobj.start_node(nodename=node): - log.error("Error starting tpu node {node}") - else: - log.info( - f"Tpu node {node} is already created, but will not start it because nodeset does not have preserve_tpu option active." - ) - else: - log.debug( - f"Will create a multi-vm TPU of type {tpuobj.node_type} tf_version {tpuobj.tf_version} in zone {tpuobj.zone} with name {node[0]}" - ) - if not tpuobj.create_node(nodename=node): - log.error("Error creating tpu node {node}") diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py deleted file mode 100644 index dafde1fea2..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/util.py +++ /dev/null @@ -1,2148 +0,0 @@ -#!/slurm/python/venv/bin/python3.13 - -# Copyright (C) SchedMD LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Iterable, List, Tuple, Optional, Any, Dict, Sequence, Type, Callable, Union -import argparse -import base64 -from dataclasses import dataclass, field -from datetime import timedelta, datetime, timezone -import hashlib -import inspect -import json -import logging -import logging.config -import logging.handlers -import math -import os -import re -import shlex -import shutil -import socket -import subprocess -import sys -from enum import Enum -from collections import defaultdict -from concurrent.futures import ThreadPoolExecutor, as_completed -from contextlib import contextmanager -from functools import lru_cache, reduce, wraps -from itertools import chain, islice -from pathlib import Path -from time import sleep, time - -# TODO: remove "type: ignore" once moved to newer version of libraries -from google.cloud import secretmanager -from google.cloud import storage # type: ignore - -import google.auth # type: ignore -from google.oauth2 import service_account # type: ignore -import googleapiclient.discovery # type: ignore -import google_auth_httplib2 # type: ignore -from googleapiclient.http import set_user_agent # type: ignore -from google.api_core.client_options import ClientOptions -import httplib2 - -import google.api_core.exceptions as gExceptions - -import requests as requests_lib - -import yaml -from addict import Dict as NSDict # type: ignore -import file_cache - -USER_AGENT = "Slurm_GCP_Scripts/1.5 (GPN:SchedMD)" -ENV_CONFIG_YAML = os.getenv("SLURM_CONFIG_YAML") -if ENV_CONFIG_YAML: - CONFIG_FILE = Path(ENV_CONFIG_YAML) -else: - CONFIG_FILE = Path(__file__).with_name("config.yaml") -API_REQ_LIMIT = 2000 - - -def mkdirp(path: Path) -> None: - path.mkdir(parents=True, exist_ok=True) - - -scripts_dir = next( - p for p in (Path(__file__).parent, Path("/slurm/scripts")) if p.is_dir() -) - - -# load all directories as Paths into a dict-like namespace -dirs = NSDict( - home = Path("/home"), - apps = Path("/opt/apps"), - slurm = Path("/slurm"), - scripts = scripts_dir, - custom_scripts = Path("/slurm/custom_scripts"), - munge = Path("/etc/munge"), - secdisk = Path("/mnt/disks/sec"), - log = Path("/var/log/slurm"), - slurm_bucket_mount = Path("/slurm/bucket"), -) - -slurmdirs = NSDict( - prefix = Path("/usr/local"), - etc = Path("/usr/local/etc/slurm"), - state = Path("/var/spool/slurm"), - key_distribution = Path("/slurm/key_distribution"), -) - - -# TODO: Remove this hack (relies on undocumented behavior of PyYAML) -# No need to represent NSDict and Path once we move to properly typed & serializable config. -yaml.SafeDumper.yaml_representers[ - None # type: ignore -] = lambda self, data: yaml.representer.SafeRepresenter.represent_str(self, str(data)) # type: ignore - - -class ApiEndpoint(Enum): - COMPUTE = "compute" - BQ = "bq" - STORAGE = "storage" - TPU = "tpu" - SECRET = "secret_manager" - - -@dataclass(frozen=True) -class AcceleratorInfo: - type: str - count: int - - @classmethod - def from_json(cls, jo: dict) -> "AcceleratorInfo": - return cls( - type=jo["guestAcceleratorType"], - count=jo["guestAcceleratorCount"]) - -@dataclass(frozen=True) -class MachineType: - name: str - guest_cpus: int - memory_mb: int - accelerators: List[AcceleratorInfo] - - @classmethod - def from_json(cls, jo: dict) -> "MachineType": - return cls( - name=jo["name"], - guest_cpus=jo["guestCpus"], - memory_mb=jo["memoryMb"], - accelerators=[ - AcceleratorInfo.from_json(a) for a in jo.get("accelerators", [])], - ) - - @property - def family(self) -> str: - # TODO: doesn't work with N1 custom machine types - # See https://cloud.google.com/compute/docs/instances/creating-instance-with-custom-machine-type#create - return self.name.split("-")[0] - - @property - def supports_smt(self) -> bool: - # https://cloud.google.com/compute/docs/cpu-platforms - if self.family in ("t2a", "t2d", "h3", "c4a", "h4d",): - return False - if self.guest_cpus == 1: - return False - return True - - @property - def sockets(self) -> int: - return { - "h3": 2, - "h4d": 2, - "c2d": 2 if self.guest_cpus > 56 else 1, - "a3": 2, - "c2": 2 if self.guest_cpus > 30 else 1, - "c3": 2 if self.guest_cpus > 88 else 1, - "c3d": 2 if self.guest_cpus > 180 else 1, - "c4": 2 if self.guest_cpus > 96 else 1, - "c4d": 2 if self.guest_cpus > 192 else 1, - }.get( - self.family, - 1, # assume 1 socket for all other families - ) - - -@dataclass(frozen=True) -class UpcomingMaintenance: - window_start_time: datetime - - @classmethod - def from_json(cls, jo: Optional[dict]) -> Optional["UpcomingMaintenance"]: - if jo is None: - return None - try: - if "windowStartTime" in jo: - ts = parse_gcp_timestamp(jo["windowStartTime"]) - elif "startTimeWindow" in jo: - ts = parse_gcp_timestamp(jo["startTimeWindow"]["earliest"]) - else: - raise Exception("Neither windowStartTime nor startTimeWindow are found") - except BaseException as e: - raise ValueError(f"Unexpected format for upcomingMaintenance: {jo}") from e - return cls(window_start_time=ts) - -@dataclass(frozen=True) -class InstanceResourceStatus: - physical_host: Optional[str] - upcoming_maintenance: Optional[UpcomingMaintenance] - - @classmethod - def from_json(cls, jo: Optional[dict]) -> "InstanceResourceStatus": - if not jo: - return cls( - physical_host=None, - upcoming_maintenance=None, - ) - - try: - maint = UpcomingMaintenance.from_json(jo.get("upcomingMaintenance")) - except ValueError as e: - log.exception("Failed to parse upcomingMaintenance, ignoring") - maint = None # intentionally swallow exception - - return cls( - physical_host=jo.get("physicalHost"), - upcoming_maintenance=maint, - ) - - -@dataclass(frozen=True) -class Instance: - name: str - zone: str - status: str - creation_timestamp: datetime - role: Optional[str] - resource_status: InstanceResourceStatus - metadata: Dict[str, str] - # TODO: use proper InstanceScheduling class - scheduling: NSDict - - @classmethod - def from_json(cls, jo: dict) -> "Instance": - return cls( - name=jo["name"], - zone=trim_self_link(jo["zone"]), - status=jo["status"], - creation_timestamp=parse_gcp_timestamp(jo["creationTimestamp"]), - resource_status=InstanceResourceStatus.from_json(jo.get("resourceStatus")), - scheduling=NSDict(jo.get("scheduling")), - role = jo.get("labels", {}).get("slurm_instance_role"), - metadata = {k["key"]: k["value"] for k in jo.get("metadata", {}).get("items", [])} - ) - - -@dataclass(frozen=True) -class NSMount: - server_ip: str - local_mount: Path - remote_mount: Path - fs_type: str - mount_options: str - -@lru_cache(maxsize=1) -def default_credentials(): - return google.auth.default()[0] - - -@lru_cache(maxsize=1) -def authentication_project(): - return google.auth.default()[1] - - -DEFAULT_UNIVERSE_DOMAIN = "googleapis.com" - - -def now() -> datetime: - """ - Return current time as timezone-aware datetime. - - IMPORTANT: DO NOT use `datetime.now()`, unless you explicitly need to have tz-naive datetime. - Otherwise there is a risk of getting: "cannot compare naive and aware datetimes" error, - since all timetstamps we receive from GCP API are tz-aware. - - Another motivation for this function is to allow to mock time in tests. - """ - return datetime.now(timezone.utc) - -def parse_gcp_timestamp(s: str) -> datetime: - """ - Parse timestamp strings returned by GCP API into datetime. - Works with both Zulu and non-Zulu timestamps. - NOTE: It always return tz-aware datetime (fallbacks to UTC and logs error). - """ - # Requires Python >= 3.7 - # TODO: Remove this "hack" of trimming the Z from timestamps once we move to Python 3.11 - # (context: https://discuss.python.org/t/parse-z-timezone-suffix-in-datetime/2220/30) - ts = datetime.fromisoformat(s.replace('Z', '+00:00')) - if ts.tzinfo is None: # fallback to UTC - log.error(f"Received timestamp without timezone info: {s}") - ts = ts.replace(tzinfo=timezone.utc) - return ts - - -def universe_domain() -> str: - try: - return instance_metadata("attributes/universe_domain") - except MetadataNotFoundError: - return DEFAULT_UNIVERSE_DOMAIN - - -def endpoint_version(api: ApiEndpoint) -> Optional[str]: - return lookup().endpoint_versions.get(api.value, None) - - -@lru_cache(maxsize=1) -def get_credentials() -> Optional[service_account.Credentials]: - """Get credentials for service account""" - key_path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") - if key_path is not None: - credentials = service_account.Credentials.from_service_account_file( - key_path, scopes=[f"https://www.{universe_domain()}/auth/cloud-platform"] - ) - else: - credentials = default_credentials() - - return credentials - - -@lru_cache(maxsize=1) -def get_dev_key() -> Optional[str]: - """Get dev key for project (uses json or yaml format)""" - try: - with open("/etc/slurm/slurm_vars.yaml", 'r') as file: - data = yaml.safe_load(file) - return data['google_developer_key'] - except: - return None - - -def create_client_options(api: ApiEndpoint) -> ClientOptions: - """Create client options for cloud endpoints""" - ver = endpoint_version(api) - ud = universe_domain() - options = {} - if ud and ud != DEFAULT_UNIVERSE_DOMAIN: - options["universe_domain"] = ud - if ver: - options["api_endpoint"] = f"https://{api.value}.{ud}/{ver}/" - co = ClientOptions(**options) - log.debug(f"Using ClientOptions = {co} for API: {api.value}") - return co - -log = logging.getLogger() - - -def access_secret_version(project_id, secret_id, version_id="latest"): - """ - Access the payload for the given secret version if one exists. The version - can be a version number as a string (e.g. "5") or an alias (e.g. "latest"). - """ - co = create_client_options(ApiEndpoint.SECRET) - client = secretmanager.SecretManagerServiceClient(client_options=co) - name = f"projects/{project_id}/secrets/{secret_id}/versions/{version_id}" - try: - response = client.access_secret_version(request={"name": name}) - log.debug(f"Secret '{name}' was found.") - payload = response.payload.data.decode("UTF-8") - except gExceptions.NotFound: - log.debug(f"Secret '{name}' was not found!") - payload = None - - return payload - - -def parse_self_link(self_link: str): - """Parse a selfLink url, extracting all useful values - https://.../v1/projects//regions//... - {'project': , 'region': , ...} - can also extract zone, instance (name), image, etc - """ - link_patt = re.compile(r"(?P[^\/\s]+)s\/(?P[^\s\/]+)") - return NSDict(link_patt.findall(self_link)) - - -def parse_bucket_uri(uri: str): - """ - Parse a bucket url - E.g. gs:/// - """ - pattern = re.compile(r"gs://(?P[^/\s]+)/(?P([^/\s]+)(/[^/\s]+)*)") - matches = pattern.match(uri) - assert matches, f"Unexpected bucker URI: '{uri}'" - return matches.group("bucket"), matches.group("path") - - -def trim_self_link(link: str): - """get resource name from self link url, eg. - https://.../v1/projects//regions/ - -> - """ - try: - return link[link.rindex("/") + 1 :] - except ValueError: - raise Exception(f"'/' not found, not a self link: '{link}' ") - - -def execute_with_futures(func, seq): - with ThreadPoolExecutor() as exe: - futures = [] - for i in seq: - future = exe.submit(func, i) - futures.append(future) - for future in as_completed(futures): - result = future.exception() - if result is not None: - raise result - - -def map_with_futures(func, seq): - with ThreadPoolExecutor() as exe: - futures = [] - for i in seq: - future = exe.submit(func, i) - futures.append(future) - for future in futures: - # Will be result or raise Exception - res = None - try: - res = future.result() - except Exception as e: - res = e - yield res - -def should_mount_slurm_bucket() -> bool: - try: - return instance_metadata("attributes/slurm_bucket_mount", silent=True).lower() == "true" - except MetadataNotFoundError: - return False - - -def _get_bucket_and_common_prefix() -> Tuple[str, str]: - uri = instance_metadata("attributes/slurm_bucket_path") - return parse_bucket_uri(uri) - -def blob_get(file): - bucket_name, path = _get_bucket_and_common_prefix() - blob_name = f"{path}/{file}" - return storage_client().get_bucket(bucket_name).blob(blob_name) - - -def blob_list(prefix="", delimiter=None): - bucket_name, path = _get_bucket_and_common_prefix() - blob_prefix = f"{path}/{prefix}" - # Note: The call returns a response only when the iterator is consumed. - blobs = storage_client().list_blobs( - bucket_name, prefix=blob_prefix, delimiter=delimiter - ) - return [blob for blob in blobs] - -def file_list(prefix="", subpath="") -> List[os.DirEntry]: - path = dirs.slurm_bucket_mount - file_prefix = f"{path}/{subpath}" - try: - files = os.scandir(file_prefix) - return [file for file in files if file.name.startswith(prefix)] - except: - return [] - # Not considering lack of file's existence as fatal (we may check for files we know don't exist). - # Responsibility of callee to determine if it is fatal or not, blob_list returns empty iterator in similar cases. - -def hash_file(fullpath: Path) -> str: - with open(fullpath, "rb") as f: - file_hash = hashlib.md5() - chunk = f.read(8192) - while chunk: - file_hash.update(chunk) - chunk = f.read(8192) - return base64.b64encode(file_hash.digest()).decode("utf-8") - - -def install_custom_scripts(check_hash:bool=False): - """download custom scripts from gcs bucket""" - role, tokens = lookup().instance_role, [] - - mounted_scripts=False - if should_mount_slurm_bucket() and role != "controller": - mounted_scripts=True - - all_prolog_tokens = ["prolog", "epilog", "task_prolog", "task_epilog"] - if role == "controller": - tokens = ["controller"] + all_prolog_tokens - elif role == "compute": - tokens = [f"nodeset-{lookup().node_nodeset_name()}"] + all_prolog_tokens - elif role == "login": - tokens = [f"login-{instance_login_group()}"] - - prefixes = [f"slurm-{tok}-script" for tok in tokens] - - # TODO: use single `blob_list`, to reduce ~4x number of GCS requests - if mounted_scripts: - source_collection = list(chain.from_iterable(file_list(prefix=p) for p in prefixes)) - else: - source_collection = list(chain.from_iterable(blob_list(prefix=p) for p in prefixes)) - - script_pattern = re.compile(r"^slurm-(?P\S+)-script-(?P\S+)") - for source in source_collection: - if mounted_scripts: - m = script_pattern.match(source.name) - else: - m = script_pattern.match(Path(source.name).name) - - if not m: - log.warning(f"found blob that doesn't match expected pattern: {source.name}") - continue - path_parts = m["path"].split("-") - path_parts[0] += ".d" - stem, _, ext = m["name"].rpartition("_") - filename = ".".join((stem, ext)) - - path = Path(*path_parts, filename) - fullpath = (dirs.custom_scripts / path).resolve() - mkdirp(fullpath.parent) - - for par in path.parents: - chown_slurm(dirs.custom_scripts / par) - need_update = True - - if check_hash and fullpath.exists() and isinstance(source,storage.Blob): - # TODO: MD5 reported by gcloud may differ from the one calculated here (e.g. if blob got gzipped), - # consider using gCRC32C - need_update = hash_file(fullpath) != source.md5_hash - - log.info(f"installing custom script: {path} from {source.name}") - - if isinstance(source,os.DirEntry): - shutil.copy(source.path, fullpath) #Needs to be copied since mounted nfs is read-only - chown_slurm(fullpath, mode=0o755) - - elif need_update: - with fullpath.open("wb") as f: - source.download_to_file(f) - chown_slurm(fullpath, mode=0o755) - -def compute_service(version="beta"): - """Make thread-safe compute service handle - creates a new Http for each request - """ - credentials = get_credentials() - dev_key = get_dev_key() - - def build_request(http, *args, **kwargs): - new_http = set_user_agent(httplib2.Http(), USER_AGENT) - if credentials is not None: - new_http = google_auth_httplib2.AuthorizedHttp(credentials, http=new_http) - return googleapiclient.http.HttpRequest(new_http, *args, **kwargs) - - ver = endpoint_version(ApiEndpoint.COMPUTE) - disc_url = googleapiclient.discovery.DISCOVERY_URI - if ver: - version = ver - disc_url = disc_url.replace(DEFAULT_UNIVERSE_DOMAIN, universe_domain()) - - log.debug(f"Using version={version} of Google Compute Engine API") - return googleapiclient.discovery.build( - "compute", - version, - requestBuilder=build_request, - credentials=credentials, - developerKey=dev_key, - discoveryServiceUrl=disc_url, - cache_discovery=False, # See https://github.com/googleapis/google-api-python-client/issues/299 - ) - -def storage_client() -> storage.Client: - """ - Config-independent storage client - """ - ud = universe_domain() - co = {} - if ud and ud != DEFAULT_UNIVERSE_DOMAIN: - co["universe_domain"] = ud - return storage.Client(client_options=ClientOptions(**co)) - - -class DeffetiveStoredConfigError(Exception): - """ - Raised when config can not be loaded and assembled from bucket - """ - pass - - -def _fill_cfg_defaults(cfg: NSDict) -> NSDict: - if not cfg.slurm_log_dir: - cfg.slurm_log_dir = dirs.log - if not cfg.slurm_bin_dir: - cfg.slurm_bin_dir = slurmdirs.prefix / "bin" - if not cfg.slurm_control_host: - try: - control_dns_name = instance_metadata("attributes/slurm_control_dns", silent=True) - cfg.slurm_control_host = control_dns_name - except MetadataNotFoundError: - cfg.slurm_control_host = f"{cfg.slurm_cluster_name}-controller" - if not cfg.slurm_control_host_port: - cfg.slurm_control_host_port = "6820-6830" - return cfg - -@dataclass -class _ConfigBlobs: - """ - "Private" class that represent a collection of GCS blobs for configuration - """ - core: storage.Blob - controller_addr: Optional[storage.Blob] - partition: List[storage.Blob] = field(default_factory=list) - nodeset: List[storage.Blob] = field(default_factory=list) - nodeset_dyn: List[storage.Blob] = field(default_factory=list) - nodeset_tpu: List[storage.Blob] = field(default_factory=list) - login_group: List[storage.Blob] = field(default_factory=list) - - @property - def hash(self) -> str: - h = hashlib.md5() - all = [self.core] + self.partition + self.nodeset + self.nodeset_dyn + self.nodeset_tpu - if self.controller_addr: - all.append(self.controller_addr) - - # sort blobs so hash is consistent - for blob in sorted(all, key=lambda b: b.name): - h.update(blob.md5_hash.encode("utf-8")) - return h.hexdigest() - -@dataclass -class _ConfigFiles: - """ - "Private" class that represent a collection of files for configuration - """ - core: Path - controller_addr: Optional[Path] - partition: List[Path] = field(default_factory=list) - nodeset: List[Path] = field(default_factory=list) - nodeset_dyn: List[Path] = field(default_factory=list) - nodeset_tpu: List[Path] = field(default_factory=list) - login_group: List[Path] = field(default_factory=list) - -def _list_config_blobs() -> _ConfigBlobs: - _, common_prefix = _get_bucket_and_common_prefix() - - core: Optional[storage.Blob] = None - controller_addr: Optional[storage.Blob] = None - rest: Dict[str, List[storage.Blob]] = {"partition": [], "nodeset": [], "nodeset_dyn": [], "nodeset_tpu": [], "login_group": []} - - is_controller = instance_role() == "controller" - - for blob in blob_list(prefix=""): - if blob.name == f"{common_prefix}/config.yaml": - core = blob - if blob.name == f"{common_prefix}/controller_addr.yaml" and not is_controller: - # Don't add this config blobs for controller to avoid "double reconfiguration": - # Initially this file doesn't exist and produce later by `setup_controller`; - # Appearance of this blob would trigger change in combined hash of config files; - # Ignore existence of this file for controller, assume that - # no other instance nodes will proceed with configuration until this file is created. - controller_addr = blob - for key in rest.keys(): - if blob.name.startswith(f"{common_prefix}/{key}_configs/"): - rest[key].append(blob) - - if core is None: - raise DeffetiveStoredConfigError(f"{common_prefix}/config.yaml not found in bucket") - - return _ConfigBlobs(core=core, controller_addr=controller_addr, **rest) - -def _list_config_files() -> _ConfigFiles: - file_dir = dirs.slurm_bucket_mount - core: Optional[Path] = None - controller_addr: Optional[Path] = None - rest: Dict[str, List[Path]] = {"partition": [], "nodeset": [], "nodeset_dyn": [], "nodeset_tpu": [], "login_group": []} - - if Path(f"{file_dir}/config.yaml").exists(): - core = Path(f"{file_dir}/config.yaml") - - for key in rest.keys(): - for f in file_list(subpath=f"{key}_configs"): - rest[key].append(f.path) - - if core is None: - raise Exception(f"config.yaml was not found in mounted folder: {dirs.slurm_bucket_mount}") #Intentionally not using DeffetiveStoredConfigError as this is considered a fatal error - - return _ConfigFiles(core=core, controller_addr=None, **rest) - -def _fetch_config(old_hash: Optional[str]) -> Optional[Tuple[NSDict, str]]: - """Fetch config from bucket, returns None if no changes are detected.""" - blobs = _list_config_blobs() - if old_hash == blobs.hash: - return None - - def _download(bs) -> List[Any]: - return [yaml.safe_load(b.download_as_text()) for b in bs] - - return _assemble_config( - core=_download([blobs.core])[0], - controller_addr=_download([blobs.controller_addr])[0] if blobs.controller_addr else None, - partitions=_download(blobs.partition), - nodesets=_download(blobs.nodeset), - nodesets_dyn=_download(blobs.nodeset_dyn), - nodesets_tpu=_download(blobs.nodeset_tpu), - login_groups=_download(blobs.login_group), - ), blobs.hash - -def _fetch_mounted_config() -> Optional[Tuple[NSDict, str]]: - if not dirs.slurm_bucket_mount.is_mount(): - raise Exception(f"{dirs.slurm_bucket_mount} is not mounted") - - files = _list_config_files() - - def _load(files) -> List[Any]: - file_yaml=[] - for file in files: - with open(file, "r") as f: - file_yaml.append(yaml.safe_load(f)) - return file_yaml - - return _assemble_config( - core=_load([files.core])[0], - controller_addr=None, - partitions=_load(files.partition), - nodesets=_load(files.nodeset), - nodesets_dyn=_load(files.nodeset_dyn), - nodesets_tpu=_load(files.nodeset_tpu), - login_groups=_load(files.login_group), - ) - -def controller_lookup_self_ip() -> str: - assert instance_role() == "controller" - # Get IP of LAST network-interface - # TODO: Consider change order of NICs definition, so right NIC is always @0. - idx = instance_metadata("network-interfaces").split()[-1] # either `0/` or `1/` - return instance_metadata(f"network-interfaces/{idx}ip") - -def _assemble_config( - core: Any, - controller_addr: Optional[Any], - partitions: List[Any], - nodesets: List[Any], - nodesets_dyn: List[Any], - nodesets_tpu: List[Any], - login_groups: List[Any], - ) -> NSDict: - cfg = NSDict(core) - - if cfg.controller_network_attachment: - # lookup controller address - if instance_role() == "controller": - # ignore stored value of `controller_addr`, it will be overwritten during `setup_controller` - cfg.slurm_control_addr = controller_lookup_self_ip() - else: - if not controller_addr: - raise DeffetiveStoredConfigError("controller_addr.yaml not found in bucket") - cfg.slurm_control_addr = controller_addr["slurm_control_addr"] - - # add partition configs - for p_yaml in partitions: - p_cfg = NSDict(p_yaml) - assert p_cfg.get("partition_name"), "partition_name is required" - p_name = p_cfg.partition_name - assert p_name not in cfg.partitions, f"partition {p_name} already defined" - cfg.partitions[p_name] = p_cfg - - # add nodeset configs - ns_names = set() - def _add_nodesets(yamls: List[Any], target: dict): - for ns_yaml in yamls: - ns_cfg = NSDict(ns_yaml) - assert ns_cfg.get("nodeset_name"), "nodeset_name is required" - ns_name = ns_cfg.nodeset_name - assert ns_name not in ns_names, f"nodeset {ns_name} already defined" - target[ns_name] = ns_cfg - ns_names.add(ns_name) - - _add_nodesets(nodesets, cfg.nodeset) - _add_nodesets(nodesets_dyn, cfg.nodeset_dyn) - _add_nodesets(nodesets_tpu, cfg.nodeset_tpu) - - # validate that configs for all referenced nodesets are present - for p in cfg.partitions.values(): - for ns_name in chain(p.partition_nodeset, p.partition_nodeset_dyn, p.partition_nodeset_tpu): - if ns_name not in ns_names: - raise DeffetiveStoredConfigError(f"nodeset {ns_name} not defined in config") - - for lg_yaml in login_groups: - lg_cfg = NSDict(lg_yaml) - assert lg_cfg.get("group_name"), "group_name is required" - lg_name = lg_cfg.group_name - assert lg_name not in cfg.login_groups - cfg.login_groups[lg_name] = lg_cfg - - if instance_role() == "login": - group = instance_login_group() - if group not in cfg.login_groups: - raise DeffetiveStoredConfigError(f"login group '{group}' does not exist in config") - - return _fill_cfg_defaults(cfg) - -def fetch_config() -> Tuple[bool, NSDict]: - """ - Fetches config from bucket and saves it locally - Returns True if new (updated) config was fetched - """ - hash_file = Path("/slurm/scripts/.config.hash") - old_hash = hash_file.read_text() if hash_file.exists() else None - - if should_mount_slurm_bucket() and instance_role() != "controller": - cfg = _fetch_mounted_config() - CONFIG_FILE.write_text(yaml.dump(cfg, Dumper=Dumper)) - chown_slurm(CONFIG_FILE) - return False, cfg - - cfg_and_hash = _fetch_config(old_hash=old_hash) - - if not cfg_and_hash: - return False, _load_config() - - cfg, hash = cfg_and_hash - hash_file.write_text(hash) - chown_slurm(hash_file) - CONFIG_FILE.write_text(yaml.dump(cfg, Dumper=Dumper)) - chown_slurm(CONFIG_FILE) - return True, cfg - -def owned_file_handler(filename): - """create file handler""" - chown_slurm(filename) - return logging.handlers.WatchedFileHandler(filename, delay=True) - -def get_log_path() -> Path: - """ - Returns path to log file for the current script. - e.g. resume.py -> /var/log/slurm/resume.log - """ - cfg_log_dir = lookup().cfg.slurm_log_dir - log_dir = Path(cfg_log_dir) if cfg_log_dir else dirs.log - return (log_dir / Path(sys.argv[0]).name).with_suffix(".log") - -def init_log_and_parse(parser: argparse.ArgumentParser) -> argparse.Namespace: - parser.add_argument( - "--debug", - "-d", - dest="loglevel", - action="store_const", - const=logging.DEBUG, - default=logging.INFO, - help="Enable debugging output", - ) - parser.add_argument( - "--trace-api", - "-t", - action="store_true", - help="Enable detailed api request output", - ) - args = parser.parse_args() - loglevel = args.loglevel - if lookup().cfg.enable_debug_logging: - loglevel = logging.DEBUG - if args.trace_api: - lookup().cfg.extra_logging_flags["trace_api"] = True - # Configure root logger - logging.config.dictConfig({ - "version": 1, - "disable_existing_loggers": True, - "formatters": { - "standard": { - "format": "%(levelname)s: %(message)s", - }, - "stamp": { - "format": "%(asctime)s %(levelname)s: %(message)s", - }, - }, - "handlers": { - "stdout_handler": { - "level": logging.DEBUG, - "formatter": "standard", - "class": "logging.StreamHandler", - "stream": sys.stdout, - }, - "file_handler": { - "()": owned_file_handler, - "level": logging.DEBUG, - "formatter": "stamp", - "filename": get_log_path(), - }, - }, - "root": { - "handlers": ["stdout_handler", "file_handler"], - "level": loglevel, - }, - }) - - sys.excepthook = _handle_exception - - return args - - -def log_api_request(request): - """log.trace info about a compute API request""" - if not lookup().cfg.extra_logging_flags.get("trace_api"): - return - # output the whole request object as pretty yaml - # the body is nested json, so load it as well - rep = json.loads(request.to_json()) - if rep.get("body", None) is not None: - rep["body"] = json.loads(rep["body"]) - pretty_req = yaml.safe_dump(rep).rstrip() - # label log message with the calling function - log.debug(f"{inspect.stack()[1].function}:\n{pretty_req}") - - -def _handle_exception(exc_type, exc_value, exc_trace): - """log exceptions other than KeyboardInterrupt""" - if not issubclass(exc_type, KeyboardInterrupt): - log.exception("Fatal exception", exc_info=(exc_type, exc_value, exc_trace)) - sys.__excepthook__(exc_type, exc_value, exc_trace) - - -def run( - args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=False, - timeout=None, - check=True, - universal_newlines=True, - **kwargs, -): - """Wrapper for subprocess.run() with convenient defaults""" - if isinstance(args, list): - args = list(filter(lambda x: x is not None, args)) - args = " ".join(args) - if not shell and isinstance(args, str): - args = shlex.split(args) - log.debug(f"run: {args}") - try: - result = subprocess.run( - args, - stdout=stdout, - stderr=stderr, - shell=shell, - timeout=timeout, - check=check, - universal_newlines=universal_newlines, - **kwargs, - ) - except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: - log_subprocess(e) - raise - log_subprocess(result) - return result - -def log_subprocess(subj: subprocess.CalledProcessError | subprocess.TimeoutExpired | subprocess.CompletedProcess) -> None: - match subj: - case subprocess.CompletedProcess(returncode=0): - # Do not log successful runs, to not overwhelm logs (e.g. scontrol show jobs --json) - # TODO: consider still doing it in DEBUG or trim output to few KBs. - return - case subprocess.CompletedProcess(): # non-zero returncode - log.error(f"Command '{subj.args}' returned exit status {subj.returncode}.") - case subprocess.CalledProcessError() | subprocess.TimeoutExpired(): - log.error(str(subj)) - - - def normalize(out: None | str | bytes) -> None | str: - """ - Turns stderr and stdout into string: - > A bytes sequence, or a string if run() was called with an encoding, errors, or text=True. None if was not captured. - """ - match out: - case None: - return None - case str(): - return out.strip() - case bytes(): - return out.decode().strip() - case _: - return repr(out) - - if stdout := normalize(subj.stdout): - log.error(f"stdout: {stdout}") - if stderr := normalize(subj.stderr): - log.error(f"stderr: {stderr}") - - -def chown_slurm(path: Path, mode=None) -> None: - if path.exists(): - if mode: - path.chmod(mode) - else: - mkdirp(path.parent) - if mode: - path.touch(mode=mode) - else: - path.touch() - try: - shutil.chown(path, user="slurm", group="slurm") - except LookupError: - log.warning(f"User 'slurm' does not exist. Cannot 'chown slurm:slurm {path}'.") - except PermissionError: - log.warning(f"Not authorized to 'chown slurm:slurm {path}'.") - except Exception as err: - log.error(err) - - -@contextmanager -def cd(path): - """Change working directory for context""" - prev = Path.cwd() - os.chdir(path) - try: - yield - finally: - os.chdir(prev) - - -def cached_property(f): - return property(lru_cache()(f)) - - -def retry(max_retries: int, init_wait_time: float, warn_msg: str, exc_type: Type[Exception]): - """Retries functions that raises the exception exc_type. - Retry time is increased by a factor of two for every iteration. - - Args: - max_retries (int): Maximum number of retries - init_wait_time (float): Initial wait time in secs - warn_msg (str): Message to print during retries - exc_type (Exception): Exception type to check for - """ - - if max_retries <= 0: - raise ValueError("Incorrect value for max_retries, must be >= 1") - if init_wait_time <= 0.0: - raise ValueError("Invalid value for init_wait_time, must be > 0.0") - - def decorator(f): - @wraps(f) - def wrapper(*args, **kwargs): - retry = 0 - secs = init_wait_time - captured_exc: Optional[BaseException] = None - while retry < max_retries: - try: - return f(*args, **kwargs) - except exc_type as e: - captured_exc = e - log.warn(f"{warn_msg}, retrying in {secs}") - sleep(secs) - retry += 1 - secs *= 2 - assert captured_exc - raise captured_exc - - return wrapper - - return decorator - - -def separate(pred: Callable[[Any], bool], coll: Iterable[Any]) -> Tuple[List[Any], List[Any]]: - """filter into 2 lists based on pred returning True or False - returns ([False], [True]) - """ - res: Tuple[List[Any], List[Any]] = ([],[]) - for el in coll: - res[pred(el)].append(el) - return res - - -def chunked(iterable, n=API_REQ_LIMIT): - """group iterator into chunks of max size n""" - it = iter(iterable) - while True: - chunk = list(islice(it, n)) - if not chunk: - return - yield chunk - -def groupby_unsorted(seq: Sequence[Any], key): - indices = defaultdict(list) - for i, el in enumerate(seq): - indices[key(el)].append(i) - for k, idxs in indices.items(): - yield k, (seq[i] for i in idxs) - - -@lru_cache(maxsize=32) -def find_ratio(a, n, s, r0=None): - """given the start (a), count (n), and sum (s), find the ratio required""" - if n == 2: - return s / a - 1 - an = a * n - if n == 1 or s == an: - return 1 - if r0 is None: - # we only need to know which side of 1 to guess, and the iteration will work - r0 = 1.1 if an < s else 0.9 - - # geometric sum formula - def f(r): - return a * (1 - r**n) / (1 - r) - s - - # derivative of f - def df(r): - rm1 = r - 1 - rn = r**n - return (a * (rn * (n * rm1 - r) + r)) / (r * rm1**2) - - MIN_DR = 0.0001 # negligible change - r = r0 - # print(f"r(0)={r0}") - MAX_TRIES = 64 - for i in range(1, MAX_TRIES + 1): - try: - dr = f(r) / df(r) - except ZeroDivisionError: - log.error(f"Failed to find ratio due to zero division! Returning r={r0}") - return r0 - r = r - dr - # print(f"r({i})={r}") - # if the change in r is small, we are close enough - if abs(dr) < MIN_DR: - break - else: - log.error(f"Could not find ratio after {MAX_TRIES}! Returning r={r0}") - return r0 - return r - - -def backoff_delay(start, timeout=None, ratio=None, count: int = 0): - """generates `count` waits starting at `start` - sum of waits is `timeout` or each one is `ratio` bigger than the last - the last wait is always 0""" - # timeout or ratio must be set but not both - assert (timeout is None) ^ (ratio is None) - assert ratio is None or ratio > 0 - assert timeout is None or timeout >= start - assert (count > 1 or timeout is not None) and isinstance(count, int) - assert start > 0 - - if count == 0: - # Equation for auto-count is tuned to have a max of - # ~int(timeout) counts with a start wait of <0.01. - # Increasing start wait decreases count eg. - # backoff_delay(10, timeout=60) -> count = 5 - count = int( - (timeout / ((start + 0.05) ** (1 / 2)) + 2) // math.log(timeout + 2) - ) - - yield start - # if ratio is set: - # timeout = start * (1 - ratio**(count - 1)) / (1 - ratio) - if ratio is None: - ratio = find_ratio(start, count - 1, timeout) - - wait = start - # we have start and 0, so we only need to generate count - 2 - for _ in range(count - 2): - wait *= ratio - yield wait - yield 0 - return - - -ROOT_URL = "http://metadata.google.internal/computeMetadata/v1" - -class MetadataNotFoundError(Exception): - pass - -def get_metadata(path:str, silent=False) -> str: - """Get metadata relative to metadata/computeMetadata/v1""" - HEADERS = {"Metadata-Flavor": "Google"} - url = f"{ROOT_URL}/{path}" - try: - resp = requests_lib.get(url, headers=HEADERS) - resp.raise_for_status() - return resp.text - except requests_lib.exceptions.HTTPError: - if not silent: - log.warning(f"metadata not found ({url})") - raise MetadataNotFoundError(f"failed to get_metadata from {url}") - - -@lru_cache(maxsize=None) -def instance_metadata(path: str, silent:bool=False) -> str: - return get_metadata(f"instance/{path}", silent=silent) - -def instance_role(): - return instance_metadata("attributes/slurm_instance_role") - - -def instance_login_group(): - return instance_metadata("attributes/slurm_login_group") - - -def natural_sort(text): - def atoi(text): - return int(text) if text.isdigit() else text - - return [atoi(w) for w in re.split(r"(\d+)", text)] - - -def to_hostlist(names: Iterable[str]) -> str: - """ - Fast implementation of `hostlist` that doesn't invoke `scontrol` - IMPORTANT: - * Acts as `scontrol show hostlistsorted`, i.e. original order is not preserved - * Achieves worse compression than `scontrol show hostlist` for some cases - """ - pref = defaultdict(list) - tokenizer = re.compile(r"^(.*?)(\d*)$") - for name in filter(None, names): - matches = tokenizer.match(name) - assert matches, name - p, s = matches.groups() - pref[p].append(s) - - def _compress_suffixes(ss: List[str]) -> List[str]: - cur, res = None, [] - - def cur_repr(): - assert cur - nums, strs = cur - if nums[0] == nums[1]: - return strs[0] - return f"{strs[0]}-{strs[1]}" - - for s in sorted(ss, key=int): - n = int(s) - if cur is None: - cur = ((n, n), (s, s)) - continue - - nums, strs = cur - if n == nums[1] + 1: - cur = ((nums[0], n), (strs[0], s)) - else: - res.append(cur_repr()) - cur = ((n, n), (s, s)) - if cur: - res.append(cur_repr()) - return res - - res = [] - for p in sorted(pref.keys()): - sl = defaultdict(list) - for s in pref[p]: - sl[len(s)].append(s) - cs = [] - for ln in sorted(sl.keys()): - if ln == 0: - res.append(p) - else: - cs.extend(_compress_suffixes(sl[ln])) - if not cs: - continue - if len(cs) == 1 and "-" not in cs[0]: - res.append(f"{p}{cs[0]}") - else: - res.append(f"{p}[{','.join(cs)}]") - return ",".join(res) - -@lru_cache(maxsize=None) -def to_hostnames(nodelist: str) -> List[str]: - """make list of hostnames from hostlist expression""" - if not nodelist: - return [] # avoid degenerate invocation of scontrol - if isinstance(nodelist, str): - hostlist = nodelist - else: - hostlist = ",".join(nodelist) - hostnames = run(f"{lookup().scontrol} show hostnames {hostlist}").stdout.splitlines() - return hostnames - - -def retry_exception(exc) -> bool: - """return true for exceptions that should always be retried""" - msg = str(exc) - retry_errors = ( - "Rate Limit Exceeded", - "Quota Exceeded", - "Quota exceeded", - ) - return any(err in msg for err in retry_errors) - - -def ensure_execute(request): - """Handle rate limits and socket time outs""" - - for retry, wait in enumerate(backoff_delay(0.5, timeout=10 * 60, count=20)): - try: - return request.execute() - except googleapiclient.errors.HttpError as e: - if retry_exception(e): - log.error(f"retry:{retry} '{e}'") - sleep(wait) - continue - raise - - except socket.timeout as e: - # socket timed out, try again - log.debug(e) - - except Exception as e: - log.error(e, exc_info=True) - raise - - break - - -def batch_execute(requests, retry_cb=None, log_err=log.error): - """execute list or dict as batch requests - retry if retry_cb returns true - """ - BATCH_LIMIT = 1000 - if not isinstance(requests, dict): - requests = {str(k): v for k, v in enumerate(requests)} # rid generated here - done = {} - failed = {} - timestamps: List[float] = [] - rate_limited = False - - def batch_callback(rid, resp, exc): - nonlocal rate_limited - if exc is not None: - log_err(f"compute request exception {rid}: {exc}") - if retry_exception(exc): - rate_limited = True - else: - req = requests.pop(rid) - failed[rid] = (req, exc) - else: - # if retry_cb is set, don't move to done until it returns false - if retry_cb is None or not retry_cb(resp): - requests.pop(rid) - done[rid] = resp - - def batch_request(reqs): - batch = lookup().compute.new_batch_http_request(callback=batch_callback) - for rid, req in reqs: - batch.add(req, request_id=rid) - return batch - - while requests: - if timestamps: - timestamps = [stamp for stamp in timestamps if stamp > time()] - if rate_limited and timestamps: - stamp = next(iter(timestamps)) - sleep(max(stamp - time(), 0)) - rate_limited = False - # up to API_REQ_LIMIT (2000) requests - # in chunks of up to BATCH_LIMIT (1000) - batches = [ - batch_request(chunk) - for chunk in chunked(islice(requests.items(), API_REQ_LIMIT), BATCH_LIMIT) - ] - timestamps.append(time() + 100) - with ThreadPoolExecutor() as exe: - futures = [] - for batch in batches: - future = exe.submit(ensure_execute, batch) - futures.append(future) - for future in futures: - result = future.exception() - if result is not None: - raise result - - return done, failed - - -def get_operation_req(lkp: "Lookup", name: str, region: Optional[str]=None, zone: Optional[str]=None) -> Any: - if zone: - return lkp.compute.zoneOperations().get(project=lkp.project, zone=zone, operation=name) - elif region: - return lkp.compute.regionOperations().get(project=lkp.project, region=region, operation=name) - return lkp.compute.globalOperations().get(project=lkp.project, operation=name) - -def wait_request(operation, project: str): - """makes the appropriate wait request for a given operation""" - if "zone" in operation: - req = lookup().compute.zoneOperations().wait( - project=project, - zone=trim_self_link(operation["zone"]), - operation=operation["name"], - ) - elif "region" in operation: - req = lookup().compute.regionOperations().wait( - project=project, - region=trim_self_link(operation["region"]), - operation=operation["name"], - ) - else: - req = lookup().compute.globalOperations().wait( - project=project, operation=operation["name"] - ) - return req - - -def wait_for_operation(operation) -> Dict[str, Any]: - """wait for given operation""" - project = parse_self_link(operation["selfLink"]).project - wait_req = wait_request(operation, project=project) - - while True: - result = ensure_execute(wait_req) - if result["status"] == "DONE": - log_errors = " with errors" if "error" in result else "" - log.debug( - f"operation complete{log_errors}: type={result['operationType']}, name={result['name']}" - ) - return result - - - -def getThreadsPerCore(template) -> int: - if not template.machine_type.supports_smt: - return 1 - return template.advancedMachineFeatures.threadsPerCore or 2 - - -@retry( - max_retries=9, - init_wait_time=1, - warn_msg="Temporary failure in name resolution", - exc_type=socket.gaierror, -) -def host_lookup(host_name: str) -> str: - return socket.gethostbyname(host_name) - - -class Dumper(yaml.SafeDumper): - """Add representers for pathlib.Path and NSDict for yaml serialization""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.add_representer(NSDict, self.represent_nsdict) - self.add_multi_representer(Path, self.represent_path) - - @staticmethod - def represent_nsdict(dumper, data): - return dumper.represent_mapping("tag:yaml.org,2002:map", data.items()) - - @staticmethod - def represent_path(dumper, path): - return dumper.represent_scalar("tag:yaml.org,2002:str", str(path)) - - -@dataclass(frozen=True) -class ReservationDetails: - project: str - zone: str - name: str - policies: List[str] # names (not URLs) of resource policies - bulk_insert_name: str # name in format suitable for bulk insert (currently identical to user supplied name in long format) - deployment_type: Optional[str] - reservation_mode: Optional[str] - - @property - def dense(self) -> bool: - return self.deployment_type == "DENSE" - - @property - def calendar(self) -> bool: - return self.reservation_mode == "CALENDAR" - -@dataclass(frozen=True) -class FutureReservation: - project: str - zone: str - name: str - specific: bool - start_time: datetime - end_time: datetime - reservation_mode: Optional[str] - active_reservation: Optional[ReservationDetails] - - @property - def calendar(self) -> bool: - return self.reservation_mode == "CALENDAR" - -@dataclass -class Job: - id: int - name: Optional[str] = None - required_nodes: Optional[str] = None - job_state: Optional[str] = None - duration: Optional[timedelta] = None - -@dataclass(frozen=True) -class NodeState: - base: str - flags: frozenset - -class Lookup: - """Wrapper class for cached data access""" - - def __init__(self, cfg): - self._cfg = cfg - - @property - def cfg(self): - return self._cfg - - @property - def project(self): - return self.cfg.project or authentication_project() - - @cached_property - def control_addr(self) -> Optional[str]: - return self.cfg.get("slurm_control_addr", None) - - @property - def control_host(self): - return self.cfg.slurm_control_host - - @cached_property - def control_host_addr(self): - return self.control_addr or host_lookup(self.cfg.slurm_control_host) - - @property - def control_host_port(self): - return self.cfg.slurm_control_host_port - - @property - def endpoint_versions(self): - return self.cfg.endpoint_versions - - @property - def scontrol(self): - return Path(self.cfg.slurm_bin_dir or "") / "scontrol" - - @cached_property - def instance_role(self): - return instance_role() - - @cached_property - def instance_role_safe(self): - try: - role = self.instance_role - except Exception as e: - log.error(e) - role = None - return role - - @property - def is_controller(self): - return self.instance_role_safe == "controller" - - @property - def is_login_node(self): - return self.instance_role_safe == "login" - - @cached_property - def compute(self): - # TODO evaluate when we need to use google_app_cred_path - if self.cfg.google_app_cred_path: - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = self.cfg.google_app_cred_path - return compute_service() - - @cached_property - def hostname(self): - return socket.gethostname() - - @cached_property - def hostname_fqdn(self): - return socket.getfqdn() - - @cached_property - def zone(self): - return instance_metadata("zone") - - node_desc_regex = re.compile( - r"^(?P(?P[^\s\-]+)-(?P\S+))-(?P(?P\w+)|(?P\[[\d,-]+\]))$" - ) - - @lru_cache(maxsize=None) - def _node_desc(self, node_name): - """Get parts from node name""" - if not node_name: - node_name = self.hostname - # workaround below is for VMs whose hostname is FQDN - node_name_short = node_name.split(".")[0] - m = self.node_desc_regex.match(node_name_short) - if not m: - raise Exception(f"node name {node_name} is not valid") - return m.groupdict() - - def node_prefix(self, node_name=None): - return self._node_desc(node_name)["prefix"] - - def node_index(self, node: str) -> int: - """ node_index("cluster-nodeset-45") == 45 """ - suff = self._node_desc(node)["suffix"] - - if suff is None: - raise ValueError(f"Node {node} name does not end with numeric index") - return int(suff) - - def node_nodeset_name(self, node_name=None): - return self._node_desc(node_name)["nodeset"] - - def node_nodeset(self, node_name=None): - nodeset_name = self.node_nodeset_name(node_name) - if nodeset_name in self.cfg.nodeset_tpu: - return self.cfg.nodeset_tpu[nodeset_name] - - return self.cfg.nodeset[nodeset_name] - - def partition_is_tpu(self, part: str) -> bool: - """check if partition with name part contains a nodeset of type tpu""" - return len(self.cfg.partitions[part].partition_nodeset_tpu) > 0 - - - def node_is_tpu(self, node_name=None): - nodeset_name = self.node_nodeset_name(node_name) - return self.cfg.nodeset_tpu.get(nodeset_name) is not None - - def node_is_fr(self, node_name:str) -> bool: - return bool(self.node_nodeset(node_name).future_reservation) - - def is_dormant_fr_node(self, node_name:str) -> bool: - fr = self.future_reservation(self.node_nodeset(node_name)) - if not fr: - return False - return fr.active_reservation is None - - def node_is_dyn(self, node_name=None) -> bool: - nodeset = self.node_nodeset_name(node_name) - return self.cfg.nodeset_dyn.get(nodeset) is not None - - def node_template(self, node_name=None) -> str: - """ Self link of nodeset template """ - return self.node_nodeset(node_name).instance_template - - def node_template_info(self, node_name=None): - return self.template_info(self.node_template(node_name)) - - def node_region(self, node_name=None): - nodeset = self.node_nodeset(node_name) - return parse_self_link(nodeset.subnetwork).region - - def nodeset_prefix(self, nodeset_name): - return f"{self.cfg.slurm_cluster_name}-{nodeset_name}" - - def nodelist_range(self, nodeset_name: str, start: int, count: int) -> str: - assert 0 <= start and 0 < count - pref = self.nodeset_prefix(nodeset_name) - if count == 1: - return f"{pref}-{start}" - return f"{pref}-[{start}-{start + count - 1}]" - - def static_dynamic_sizes(self, nodeset: NSDict) -> Tuple[int, int]: - return (nodeset.node_count_static or 0, nodeset.node_count_dynamic_max or 0) - - def nodelist(self, nodeset) -> str: - cnt = sum(self.static_dynamic_sizes(nodeset)) - if cnt == 0: - return "" - return self.nodelist_range(nodeset.nodeset_name, 0, cnt) - - def nodenames(self, nodeset) -> Tuple[Iterable[str], Iterable[str]]: - pref = self.nodeset_prefix(nodeset.nodeset_name) - s_count, d_count = self.static_dynamic_sizes(nodeset) - return ( - (f"{pref}-{i}" for i in range(s_count)), - (f"{pref}-{i}" for i in range(s_count, s_count + d_count)), - ) - - def power_managed_nodesets(self) -> Iterable[NSDict]: - return chain(self.cfg.nodeset.values(), self.cfg.nodeset_tpu.values()) - - def is_power_managed_node(self, node_name: str) -> bool: - try: - ns = self.node_nodeset(node_name) - if ns is None: - return False - idx = int(self._node_desc(node_name)["suffix"]) - return idx < sum(self.static_dynamic_sizes(ns)) - except Exception: - return False - - def is_static_node(self, node_name: str) -> bool: - if not self.is_power_managed_node(node_name): - return False - idx = int(self._node_desc(node_name)["suffix"]) - return idx < self.node_nodeset(node_name).node_count_static - - @lru_cache(maxsize=None) - def slurm_nodes(self) -> Dict[str, NodeState]: - def parse_line(node_line) -> Tuple[str, NodeState]: - """turn node,state line to (node, NodeState)""" - # state flags include: CLOUD, COMPLETING, DRAIN, FAIL, POWERED_DOWN, - # POWERING_DOWN - node, fullstate = node_line.split(",") - state = fullstate.split("+") - state_tuple = NodeState(base=state[0], flags=frozenset(state[1:])) - return (node, state_tuple) - - cmd = ( - f"{self.scontrol} show nodes | " - r"grep -oP '^NodeName=\K(\S+)|\s+State=\K(\S+)' | " - r"paste -sd',\n'" - ) - node_lines = run(cmd, shell=True).stdout.rstrip().splitlines() - nodes = { - node: state - for node, state in map(parse_line, node_lines) - if "CLOUD" in state.flags or "DYNAMIC_NORM" in state.flags - } - return nodes - - def node_state(self, nodename: str) -> Optional[NodeState]: - state = self.slurm_nodes().get(nodename) - if state is not None: - return state - - # state is None => Slurm doesn't know this node, - # there are two reasons: - # * happy: - # * node belongs to removed nodeset - # * node belongs to downsized portion of nodeset - # * dynamic node that didn't register itself - # * unhappy: - # * there is a drift in Slurm and SlurmGCP configurations - # * `slurm_nodes` function failed to handle `scontrol show nodes`, - # TODO: make `slurm_nodes` robust by using `scontrol show nodes --json` - # In either of "unhappy" cases it's too dangerous to proceed - abort slurmsync. - try: - ns = self.node_nodeset(nodename) - except: - log.info(f"Unknown node {nodename}, belongs to unknown nodeset") - return None # Can't find nodeset, may be belongs to removed nodeset - - if self.node_is_dyn(nodename): - log.info(f"Unknown node {nodename}, belongs to dynamic nodeset") - return None # we can't make any judjment for dynamic nodes - - cnt = sum(self.static_dynamic_sizes(ns)) - if self.node_index(nodename) >= cnt: - log.info(f"Unknown node {nodename}, out of nodeset size boundaries ({cnt})") - return None # node belongs to downsized nodeset - - raise RuntimeError(f"Slurm does not recognize node {nodename}, potential misconfiguration.") - - - @lru_cache(maxsize=1) - def instances(self) -> Dict[str, Instance]: - instance_information_fields = [ - "creationTimestamp", - "name", - "resourceStatus", - "scheduling", - "status", - "labels.slurm_instance_role", - "zone", - "metadata", - ] - - instance_fields = ",".join(sorted(instance_information_fields)) - fields = f"items.zones.instances({instance_fields}),nextPageToken" - flt = f"labels.slurm_cluster_name={self.cfg.slurm_cluster_name} AND name:{self.cfg.slurm_cluster_name}-*" - act = self.compute.instances() - op = act.aggregatedList(project=self.project, fields=fields, filter=flt) - - instances = {} - while op is not None: - result = ensure_execute(op) - for zone in result.get("items", {}).values(): - for jo in zone.get("instances", []): - inst = Instance.from_json(jo) - if inst.name in instances: - log.error(f"Duplicate VM name {inst.name} across multiple zones") - instances[inst.name] = inst - op = act.aggregatedList_next(op, result) - return instances - - def instance(self, instance_name: str) -> Optional[Instance]: - return self.instances().get(instance_name) - - @lru_cache() - def _get_reservation(self, project: str, zone: str, name: str) -> Any: - """See https://cloud.google.com/compute/docs/reference/rest/v1/reservations""" - return self.compute.reservations().get( - project=project, zone=zone, reservation=name).execute() - - @lru_cache() - def get_mig(self, project: str, zone: str, self_link:str) -> Any: - """https://cloud.google.com/compute/docs/reference/rest/v1/instanceGroupManagers""" - return self.compute.instanceGroupManagers().get(project=project, zone=zone, instanceGroupManager=self_link).execute() - - @lru_cache - def get_mig_instances(self, project: str, zone: str, self_link:str) -> Any: - return self.compute.instanceGroupManagers().listManagedInstances(project=project, zone=zone, instanceGroupManager=self_link).execute() - - @lru_cache() - def get_mig_list(self, project: str, zone: str) -> Any: - """https://cloud.google.com/compute/docs/reference/rest/v1/instanceGroupManagers""" - return self.compute.instanceGroupManagers().list(project=project, zone=zone).execute() - - @lru_cache() - def _get_future_reservation(self, project:str, zone:str, name: str) -> Any: - """See https://cloud.google.com/compute/docs/reference/rest/v1/futureReservations""" - return self.compute.futureReservations().get(project=project, zone=zone, futureReservation=name).execute() - - def get_reservation_details(self, project:str, zone:str, name:str, bulk_insert_name:str) -> ReservationDetails: - reservation = self._get_reservation(project, zone, name) - - # Converts policy URLs to names, e.g.: - # projects/111111/regions/us-central1/resourcePolicies/zebra -> zebra - policies = [u.split("/")[-1] for u in reservation.get("resourcePolicies", {}).values()] - - return ReservationDetails( - project=project, - zone=zone, - name=name, - policies=policies, - deployment_type=reservation.get("deploymentType"), - reservation_mode=reservation.get("reservationMode"), - bulk_insert_name=bulk_insert_name) - - def nodeset_reservation(self, nodeset: NSDict) -> Optional[ReservationDetails]: - if not nodeset.reservation_name: - return None - - zones = list(nodeset.zone_policy_allow or []) - assert len(zones) == 1, "Only single zone is supported if using a reservation" - zone = zones[0] - - regex = re.compile(r'^projects/(?P[^/]+)/reservations/(?P[^/]+)(/.*)?$') - if not (match := regex.match(nodeset.reservation_name)): - raise ValueError( - f"Invalid reservation name: '{nodeset.reservation_name}', expected format is 'projects/PROJECT/reservations/NAME'" - ) - - project, name = match.group("project", "reservation") - return self.get_reservation_details(project, zone, name, nodeset.reservation_name) - - def future_reservation(self, nodeset: NSDict) -> Optional[FutureReservation]: - if not nodeset.future_reservation: - return None - - active_reservation = None - match = re.search(r'^projects/(?P[^/]+)/zones/(?P[^/]+)/futureReservations/(?P[^/]+)(/.*)?$', nodeset.future_reservation) - assert match, f"Invalid future reservation name '{nodeset.future_reservation}'" - project, zone, name = match.group("project","zone","name") - fr = self._get_future_reservation(project,zone,name) - - start_time = parse_gcp_timestamp(fr["timeWindow"]["startTime"]) - end_time = parse_gcp_timestamp(fr["timeWindow"]["endTime"]) - - if "autoCreatedReservations" in fr["status"] and (res:=fr["status"]["autoCreatedReservations"][0]): - if start_time <= now() <=end_time: - match = re.search(r'projects/(?P[^/]+)/zones/(?P[^/]+)/reservations/(?P[^/]+)(/.*)?$',res) - assert match, f"Unexpected reservation name '{res}'" - res_name = match.group("name") - bulk_insert_name = f"projects/{project}/reservations/{res_name}" - active_reservation = self.get_reservation_details(project, zone, res_name, bulk_insert_name) - - return FutureReservation( - project=project, - zone=zone, - name=name, - specific=fr["specificReservationRequired"], - start_time=start_time, - end_time=end_time, - reservation_mode=fr.get("reservationMode"), - active_reservation=active_reservation - ) - - @lru_cache(maxsize=1) - def machine_types(self): - field_names = "name,zone,guestCpus,memoryMb,accelerators" - fields = f"items.zones.machineTypes({field_names}),nextPageToken" - - machines: Dict[str, Dict[str, Any]] = defaultdict(dict) - act = self.compute.machineTypes() - op = act.aggregatedList(project=self.project, fields=fields) - while op is not None: - result = ensure_execute(op) - machine_iter = chain.from_iterable( - scope.get("machineTypes", []) for scope in result["items"].values() - ) - for machine in machine_iter: - name = machine["name"] - zone = machine["zone"] - machines[name][zone] = machine - - op = act.aggregatedList_next(op, result) - return machines - - def machine_type(self, name: str) -> MachineType: - custom_patt = re.compile( - r"((?P\w+)-)?custom-(?P\d+)-(?P\d+)" - ) - if match := custom_patt.match(name): - return MachineType( - name=name, - guest_cpus=int(match.group("cpus")), - memory_mb=int(match.group("mem")), - accelerators=[], - ) - - machines = self.machine_types() - if name not in machines: - raise Exception(f"machine type {name} not found") - per_zone = machines[name] - assert per_zone - return MachineType.from_json( - next(iter(per_zone.values())) # pick the first/any zone - ) - - def template_machine_conf(self, template_link): - template = self.template_info(template_link) - machine = template.machine_type - - machine_conf = NSDict() - machine_conf.boards = 1 # No information, assume 1 - machine_conf.sockets = machine.sockets - # the value below for SocketsPerBoard must be type int - machine_conf.sockets_per_board = machine_conf.sockets // machine_conf.boards - machine_conf.threads_per_core = 1 - _div = 2 if getThreadsPerCore(template) == 1 else 1 - machine_conf.cpus = ( - int(machine.guest_cpus / _div) if machine.supports_smt else machine.guest_cpus - ) - machine_conf.cores_per_socket = int(machine_conf.cpus / machine_conf.sockets) - # Because the actual memory on the host will be different than - # what is configured (e.g. kernel will take it). From - # experiments, about 16 MB per GB are used (plus about 400 MB - # buffer for the first couple of GB's. Using 30 MB to be safe. - gb = machine.memory_mb // 1024 - machine_conf.memory = machine.memory_mb - (400 + (30 * gb)) - return machine_conf - - @lru_cache(maxsize=None) - def template_info(self, template_link): - template_name = trim_self_link(template_link) - cache = file_cache.cache("template_cache") - - if cached := cache.get(template_name): - return NSDict(cached) - - template = ensure_execute( - self.compute.instanceTemplates().get( - project=self.project, instanceTemplate=template_name - ) - ).get("properties") - template = NSDict(template) - # name and link are not in properties, so stick them in - template.name = template_name - template.link = template_link - template.machine_type = self.machine_type(template.machineType) - # TODO delete metadata to reduce memory footprint? - # del template.metadata - - # translate gpus into an easier-to-read format - if template.machine_type.accelerators: - template.gpu = template.machine_type.accelerators[0] - elif template.guestAccelerators: - tga = template.guestAccelerators[0] - template.gpu = AcceleratorInfo( - type=tga.acceleratorType, - count=tga.acceleratorCount) - else: - template.gpu = None - - cache.set(template_name, template.to_dict()) - return template - - - def _parse_job_info(self, job_info: str) -> Job: - """Extract job details""" - if match:= re.search(r"JobId=(\d+)", job_info): - job_id = int(match.group(1)) - else: - raise ValueError(f"Job ID not found in the job info: {job_info}") - - if match:= re.search(r"TimeLimit=(?:(\d+)-)?(\d{2}):(\d{2}):(\d{2})", job_info): - days, hours, minutes, seconds = match.groups() - duration = timedelta( - days=int(days) if days else 0, - hours=int(hours), - minutes=int(minutes), - seconds=int(seconds) - ) - else: - duration = None - - if match := re.search(r"JobName=([^\n]+)", job_info): - name = match.group(1) - else: - name = None - - if match := re.search(r"JobState=(\w+)", job_info): - job_state = match.group(1) - else: - job_state = None - - if match := re.search(r"ReqNodeList=([^ ]+)", job_info): - required_nodes = match.group(1) - else: - required_nodes = None - - return Job(id=job_id, duration=duration, name=name, job_state=job_state, required_nodes=required_nodes) - - @lru_cache - def get_jobs(self) -> List[Job]: - res = run(f"{self.scontrol} show jobs", timeout=30) - - return [self._parse_job_info(job) for job in res.stdout.split("\n\n")[:-1]] - - @lru_cache - def job(self, job_id: int) -> Optional[Job]: - job_info = run(f"{self.scontrol} show jobid {job_id}", check=False).stdout.rstrip() - if not job_info: - return None - - return self._parse_job_info(job_info=job_info) - - @property - def etc_dir(self) -> Path: - return Path(self.cfg.output_dir or slurmdirs.etc) - - def controller_mount_server_ip(self) -> str: - return self.control_addr or self.control_host - - def normalize_ns_mount(self, ns: Union[dict, NSMount]) -> NSMount: - if isinstance(ns, NSMount): - return ns - - server_ip = ns.get("server_ip") or "$controller" - if server_ip == "$controller": - server_ip = self.controller_mount_server_ip() - - return NSMount( - server_ip=server_ip, - local_mount=Path(ns["local_mount"]), - remote_mount=Path(ns["remote_mount"]), - fs_type=ns["fs_type"], - mount_options=ns["mount_options"], - ) - - @property - def munge_mount(self) -> NSMount: - if self.cfg.munge_mount: - mnt = self.cfg.munge_mount - mnt.local_mount = mnt.local_mount or "/mnt/munge" - return self.normalize_ns_mount(mnt) - else: - return NSMount( - server_ip=self.controller_mount_server_ip(), - local_mount=Path("/mnt/munge"), - remote_mount=dirs.munge, - fs_type="nfs", - mount_options="defaults,hard,intr,_netdev", - ) - - @property - def slurm_key_mount(self) -> NSMount: - if self.cfg.slurm_key_mount: - mnt = self.cfg.slurm_key_mount - mnt.local_mount = mnt.local_mount or slurmdirs.key_distribution - return self.normalize_ns_mount(mnt) - else: - return NSMount( - server_ip=self.controller_mount_server_ip(), - local_mount=slurmdirs.key_distribution, - remote_mount=slurmdirs.key_distribution, - fs_type="nfs", - mount_options="defaults,hard,intr,_netdev", - ) - - def is_flex_node(self, node: str) -> bool: - try: - nodeset = self.node_nodeset(node) - if nodeset.dws_flex.use_bulk_insert: - return False #For legacy flex support - return bool(nodeset.dws_flex.enabled) - except: - return False - - def is_provisioning_flex_node(self, node:str) -> bool: - if not self.is_flex_node(node): - return False - if self.instance(node) is not None: - return True - - nodeset = self.node_nodeset(node) - zones = nodeset.zone_policy_allow - assert len(zones) == 1 - zone = zones[0] - - potential_migs=[] - mig_list=self.get_mig_list(self.project, zone) - - if not mig_list or not mig_list.get("items"): - return False - - for mig in mig_list["items"]: - if not mig.get("instanceTemplate"): #possibly an old MIG - return False - if mig["instanceTemplate"] == self.node_template(node) and mig["currentActions"]["creating"] > 0: - potential_migs.append(self.get_mig_instances(self.project, zone, trim_self_link(mig["selfLink"]))) - - if not potential_migs: - return False - - for instance_collection in potential_migs[0]["managedInstances"]: - if node in instance_collection["name"] and instance_collection["currentAction"]=="CREATING": - return True - return False - - def cluster_regions(self) -> list[str]: - """ - Returns all regions used in cluster - NOTE: only concerned with normal nodesets, - neither TPU, nor dynamic, nor login node, nor controller node are considered - """ - res = set() - for nodeset in self.cfg.nodeset.values(): - res.add(parse_self_link(nodeset.subnetwork).region) - return list(res) - - - -_lkp: Optional[Lookup] = None - -def _load_config() -> NSDict: - return NSDict(yaml.safe_load(CONFIG_FILE.read_text())) - -def lookup() -> Lookup: - global _lkp - if _lkp is None: - try: - cfg = _load_config() - except FileNotFoundError: - log.error(f"config file not found: {CONFIG_FILE}") - cfg = NSDict() # TODO: fail here, once all code paths are covered (mainly init_logging) - _lkp = Lookup(cfg) - return _lkp - -def update_config(cfg: NSDict) -> None: - global _lkp - _lkp = Lookup(cfg) - -def scontrol_reconfigure(lkp: Lookup) -> None: - log.info("Running scontrol reconfigure") - run(f"{lkp.scontrol} reconfigure") diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py deleted file mode 100644 index d1d77a1833..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/scripts/watch_delete_vm_op.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Any - - -from dataclasses import dataclass, asdict -import util -import local_pubsub - -import logging -log = logging.getLogger() - -# Name of the topic -TOPIC = "watch_delete_vm_op" - -@dataclass(frozen=True) -class WatchDeleteVmOp_Message: - op_name: str - zone: str - node: str - -class WatchDeleteVmOp_Topic: - def __init__(self, topic: local_pubsub.Topic) -> None: - self._t = topic - - def publish(self, op: dict[str, Any], node: str) -> None: - assert op.get("operationType") == "delete" - assert op.get("zone") - assert node - - msg = WatchDeleteVmOp_Message(op_name=op["name"], zone=op["zone"], node=node) - self._t.publish(data=asdict(msg)) - - -def watch_delete_vm_op_topic() -> WatchDeleteVmOp_Topic: - return WatchDeleteVmOp_Topic(local_pubsub.topic(TOPIC)) - - -def _watch_op(lkp: util.Lookup, m: WatchDeleteVmOp_Message) -> bool: - """ - Processes VM delete-operation. - If operation is still running - do nothing - If operation failed - log error & remove op from watch list - If operation is done - remove op from watch list do nothing - - To avoid querying status for each op individually, use list of VM instances as - a source of data. Don't query op for instance X if instance X is not present - (presumably deleted). - NOTE: This optimization can lead to false-positives - - absence of error-logs in case op failed, but VM got deleted by other means. - - Returns True if message should be marked as processed (ack). - """ - - inst = lkp.instance(m.node) - - if not inst: - log.debug(f"Stop watching op {m.op_name}, VM {m.node} appears to be deleted") - return True # ack, potentially false-positive - - if inst.status == "TERMINATED": - log.debug(f"Stop watching op {m.op_name}, VM {m.node} is TERMINATED") - return True # ack, potentially false-positive - - if inst.status == "STOPPING": - log.debug(f"Skipping op {m.op_name}, VM {m.node} is STOPPING") - return False # try later - - try: - op = util.get_operation_req(lkp, m.op_name, zone=m.zone).execute() - except: - # TODO: consider less conservative handling, but be careful not to cause deadlettering. - log.exception(f"Failed to get operation {m.op_name}, will not retry") - return True # ack (remove) - - if op["status"] != "DONE": - log.debug(f"Watching op {m.op_name} is still not done ({op['status']})") - return False # try later - - if "error" in op: - log.error(f"Operation {m.op_name} to delete {m.node} finished with error: {op['error']}") - else: - log.debug(f"Operation {m.op_name} to delete {m.node} successfully finished") - return True # ack - - -def watch_vm_delete_ops(lkp: util.Lookup) -> None: - sub = local_pubsub.subscription(TOPIC) - - # Pull once instead of "pulling until empty", motivation: - # Bulk of cases processed by `_watch_op` relies on freshness of `lkp.instances`, - # `lkp.instances` are fetched once during run of `slurmsync`. - # Therefore we shouldn't try to re-process messages that has been already NACKed in this run, - # since they will be handled with the same `lkp.instance` as a previous attempt. - msgs = sub.pull(max_messages=1000) # 1000 is arbitrary number to be adjusted if needed. - log.debug(f"Processing {len(msgs)} delete VM operations") - # TODO: handle messages in butches to improve latency - for m in msgs: - try: - dm = WatchDeleteVmOp_Message(**m.data) - ack = _watch_op(lkp, dm) - except Exception: - log.exception(f"Failed to process the message {m.id}, removing") - ack = True - if ack: - sub.ack([m.id]) - else: - sub.modify_ack_deadline([m.id], deadline=0) # NACK - - - - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf deleted file mode 100644 index 7163ec4c60..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/modules/slurm_files/variables.tf +++ /dev/null @@ -1,500 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "bucket_name" { - description = <<-EOD - Name of GCS bucket to use. - EOD - type = string -} - -variable "bucket_dir" { - description = "Bucket directory for cluster files to be put into." - type = string - default = null -} - -variable "enable_debug_logging" { - type = bool - description = "Enables debug logging mode. Not for production use." - default = false -} - -variable "extra_logging_flags" { - type = map(bool) - description = "The only available flag is `trace_api`" - default = {} -} - -variable "project_id" { - description = "The GCP project ID." - type = string -} - -variable "enable_slurm_auth" { - description = < x... } - nodeset_map = { for k, vs in local.nodeset_map_ell : k => vs[0] } - - nodeset_tpu_map_ell = { for x in var.nodeset_tpu : x.nodeset_name => x... } - nodeset_tpu_map = { for k, vs in local.nodeset_tpu_map_ell : k => vs[0] } - - nodeset_dyn_map_ell = { for x in var.nodeset_dyn : x.nodeset_name => x... } - nodeset_dyn_map = { for k, vs in local.nodeset_dyn_map_ell : k => vs[0] } - - - no_reservation_affinity = { type : "NO_RESERVATION" } -} - -# NODESET -module "slurm_nodeset_template" { - source = "../../internal/slurm-gcp/instance_template" - for_each = local.nodeset_map - - project_id = var.project_id - slurm_cluster_name = local.slurm_cluster_name - slurm_instance_role = "compute" - slurm_bucket_path = module.slurm_files.slurm_bucket_path - - additional_disks = each.value.additional_disks - bandwidth_tier = each.value.bandwidth_tier - can_ip_forward = each.value.can_ip_forward - advanced_machine_features = each.value.advanced_machine_features - disk_auto_delete = each.value.disk_auto_delete - disk_labels = each.value.disk_labels - disk_resource_manager_tags = each.value.disk_resource_manager_tags - disk_size_gb = each.value.disk_size_gb - disk_type = each.value.disk_type - enable_confidential_vm = each.value.enable_confidential_vm - enable_oslogin = each.value.enable_oslogin - enable_shielded_vm = each.value.enable_shielded_vm - gpu = each.value.gpu - labels = merge(each.value.labels, { slurm_nodeset = each.value.nodeset_name }) - machine_type = each.value.machine_type - metadata = merge(each.value.metadata, local.universe_domain) - min_cpu_platform = each.value.min_cpu_platform - name_prefix = each.value.nodeset_name - on_host_maintenance = each.value.on_host_maintenance - preemptible = each.value.preemptible - region = each.value.region - resource_manager_tags = each.value.resource_manager_tags - spot = each.value.spot - termination_action = each.value.termination_action - service_account = each.value.service_account - shielded_instance_config = each.value.shielded_instance_config - source_image_family = each.value.source_image_family - source_image_project = each.value.source_image_project - source_image = each.value.source_image - subnetwork = each.value.subnetwork_self_link - additional_networks = each.value.additional_networks - access_config = each.value.access_config - tags = concat([local.slurm_cluster_name], each.value.tags) - - max_run_duration = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? each.value.dws_flex.max_run_duration : null - provisioning_model = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? "FLEX_START" : null - reservation_affinity = (each.value.dws_flex.enabled && !each.value.dws_flex.use_bulk_insert) ? local.no_reservation_affinity : null -} - -module "nodeset_cleanup" { - source = "./modules/cleanup_compute" - for_each = local.nodeset_map - - nodeset = each.value - project_id = var.project_id - slurm_cluster_name = local.slurm_cluster_name - enable_cleanup_compute = var.enable_cleanup_compute - universe_domain = var.universe_domain - endpoint_versions = var.endpoint_versions - gcloud_path_override = var.gcloud_path_override - nodeset_template = module.slurm_nodeset_template[each.value.nodeset_name].self_link -} - -locals { - nodesets = [for name, ns in local.nodeset_map : { - nodeset_name = ns.nodeset_name - node_conf = ns.node_conf - dws_flex = ns.dws_flex - instance_template = module.slurm_nodeset_template[ns.nodeset_name].self_link - node_count_dynamic_max = ns.node_count_dynamic_max - node_count_static = ns.node_count_static - subnetwork = ns.subnetwork_self_link - reservation_name = ns.reservation_name - future_reservation = ns.future_reservation - maintenance_interval = ns.maintenance_interval - instance_properties_json = ns.instance_properties_json - enable_placement = ns.enable_placement - placement_max_distance = ns.placement_max_distance - network_storage = ns.network_storage - zone_target_shape = ns.zone_target_shape - zone_policy_allow = ns.zone_policy_allow - zone_policy_deny = ns.zone_policy_deny - enable_maintenance_reservation = ns.enable_maintenance_reservation - enable_opportunistic_maintenance = ns.enable_opportunistic_maintenance - }] -} - -# NODESET TPU -module "slurm_nodeset_tpu" { - source = "../../internal/slurm-gcp/nodeset_tpu" - for_each = local.nodeset_tpu_map - - project_id = var.project_id - node_count_dynamic_max = each.value.node_count_dynamic_max - node_count_static = each.value.node_count_static - nodeset_name = each.value.nodeset_name - zone = each.value.zone - node_type = each.value.node_type - accelerator_config = each.value.accelerator_config - tf_version = each.value.tf_version - preemptible = each.value.preemptible - preserve_tpu = each.value.preserve_tpu - enable_public_ip = each.value.enable_public_ip - service_account = each.value.service_account - data_disks = each.value.data_disks - docker_image = each.value.docker_image - subnetwork = each.value.subnetwork -} - -module "nodeset_cleanup_tpu" { - source = "./modules/cleanup_tpu" - for_each = local.nodeset_tpu_map - - nodeset = { - nodeset_name = each.value.nodeset_name - zone = each.value.zone - } - - project_id = var.project_id - slurm_cluster_name = local.slurm_cluster_name - enable_cleanup_compute = var.enable_cleanup_compute - universe_domain = var.universe_domain - endpoint_versions = var.endpoint_versions - gcloud_path_override = var.gcloud_path_override - - depends_on = [ - # Depend on controller network, as a best effort to avoid - # subnetwork resourceInUseByAnotherResource error - var.subnetwork_self_link - ] -} - -resource "google_storage_bucket_object" "parition_config" { - for_each = { for p in var.partitions : p.partition_name => p } - - bucket = module.slurm_files.bucket_name - name = "${module.slurm_files.bucket_dir}/partition_configs/${each.key}.yaml" - content = yamlencode(each.value) -} - -moved { - from = module.slurm_files.google_storage_bucket_object.parition_config - to = google_storage_bucket_object.parition_config -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf deleted file mode 100644 index 32a7be1ad4..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/slurm_files.tf +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# BUCKET - -locals { - synt_suffix = substr(md5("${local.controller_project_id}${var.deployment_name}"), 0, 5) - synth_bucket_name = "${local.slurm_cluster_name}${local.synt_suffix}" - - bucket_name = var.create_bucket ? module.bucket[0].name : var.bucket_name -} - -module "bucket" { - source = "terraform-google-modules/cloud-storage/google" - version = "~> 6.1" - - count = var.create_bucket ? 1 : 0 - - location = var.region - names = [local.synth_bucket_name] - prefix = "slurm" - project_id = local.controller_project_id - - force_destroy = { - (local.synth_bucket_name) = true - } - - labels = merge(local.labels, { - slurm_cluster_name = local.slurm_cluster_name - }) -} - -# BUCKET IAMs -locals { - compute_sa = toset(flatten([for x in module.slurm_nodeset_template : x.service_account])) - compute_tpu_sa = toset(flatten([for x in module.slurm_nodeset_tpu : x.service_account])) - login_sa = toset(flatten([for x in module.login : x.service_account])) - - viewers = toset(flatten([ - "serviceAccount:${module.slurm_controller_template.service_account.email}", - formatlist("serviceAccount:%s", [for x in local.compute_sa : x.email]), - formatlist("serviceAccount:%s", [for x in local.compute_tpu_sa : x.email if x.email != null]), - formatlist("serviceAccount:%s", [for x in local.login_sa : x.email]), - ])) -} - - -resource "google_storage_bucket_iam_member" "viewers" { - for_each = local.viewers - bucket = local.bucket_name - role = "roles/storage.objectViewer" - member = each.value -} - -resource "google_storage_bucket_iam_member" "legacy_readers" { - for_each = local.viewers - bucket = local.bucket_name - role = "roles/storage.legacyBucketReader" - member = each.value -} - -locals { - daos_ns = [ - for ns in var.network_storage : - ns if ns.fs_type == "daos" - ] - - daos_client_install_runners = [ - for ns in local.daos_ns : - ns.client_install_runner if ns.client_install_runner != null - ] - - daos_mount_runners = [ - for ns in local.daos_ns : - ns.mount_runner if ns.mount_runner != null - ] - - daos_network_storage_runners = concat( - local.daos_client_install_runners, - local.daos_mount_runners, - ) - - daos_install_mount_script = { - filename = "ghpc_daos_mount.sh" - content = length(local.daos_ns) > 0 ? module.daos_network_storage_scripts[0].startup_script : "" - } - - common_scripts = length(local.daos_ns) > 0 ? [local.daos_install_mount_script] : [] -} - -# SLURM FILES -locals { - ghpc_startup_script_controller = concat( - local.common_scripts, - [{ - filename = "ghpc_startup.sh" - content = var.controller_startup_script - }]) - - controller_state_disk = { - device_name : try(google_compute_disk.controller_disk[0].name, null) - } - - - nodeset_startup_scripts = { for k, v in local.nodeset_map : k => concat(local.common_scripts, v.startup_script) } -} - -module "daos_network_storage_scripts" { - count = length(local.daos_ns) > 0 ? 1 : 0 - - source = "../../../../modules/scripts/startup-script" - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.daos_network_storage_runners -} - -module "slurm_files" { - source = "./modules/slurm_files" - - project_id = var.project_id - slurm_cluster_name = local.slurm_cluster_name - bucket_dir = var.bucket_dir - bucket_name = local.bucket_name - controller_network_attachment = var.controller_network_attachment - - slurmdbd_conf_tpl = var.slurmdbd_conf_tpl - slurm_conf_tpl = var.slurm_conf_tpl - slurm_conf_template = var.slurm_conf_template - cgroup_conf_tpl = var.cgroup_conf_tpl - cloud_parameters = var.cloud_parameters - cloudsql_secret = try( - one(google_secret_manager_secret_version.cloudsql_version[*].id), - null) - - controller_startup_scripts = local.ghpc_startup_script_controller - controller_startup_scripts_timeout = var.controller_startup_scripts_timeout - nodeset_startup_scripts = local.nodeset_startup_scripts - compute_startup_scripts_timeout = var.compute_startup_scripts_timeout - controller_state_disk = local.controller_state_disk - - enable_debug_logging = var.enable_debug_logging - extra_logging_flags = var.extra_logging_flags - - enable_slurm_auth = var.enable_slurm_auth - - enable_bigquery_load = var.enable_bigquery_load - enable_external_prolog_epilog = var.enable_external_prolog_epilog - enable_chs_gpu_health_check_prolog = var.enable_chs_gpu_health_check_prolog - enable_chs_gpu_health_check_epilog = var.enable_chs_gpu_health_check_epilog - epilog_scripts = var.epilog_scripts - prolog_scripts = var.prolog_scripts - task_epilog_scripts = var.task_epilog_scripts - task_prolog_scripts = var.task_prolog_scripts - - disable_default_mounts = !var.enable_default_mounts - network_storage = [ - for storage in var.network_storage : { - server_ip = storage.server_ip, - remote_mount = storage.remote_mount, - local_mount = storage.local_mount, - fs_type = storage.fs_type, - mount_options = storage.mount_options - } - if storage.fs_type != "daos" - ] - - nodeset = local.nodesets - nodeset_dyn = values(local.nodeset_dyn_map) - # Use legacy format for now - nodeset_tpu = values(module.slurm_nodeset_tpu)[*] - - - depends_on = [module.bucket] - - # Providers - endpoint_versions = var.endpoint_versions -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf deleted file mode 100644 index 0641cea7a8..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/source_image_logic.tf +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # Currently supported images and projects - known_project_families = { - schedmd-slurm-public = [ - "slurm-gcp-6-9-debian-11", - "slurm-gcp-6-9-hpc-rocky-linux-8", - "slurm-gcp-6-9-ubuntu-2004-lts", - "slurm-gcp-6-9-ubuntu-2204-lts-arm64" - ] - } - - # This approach to "hacking" the project name allows a chain of Terraform - # calls to set the instance source_image (boot disk) with a "relative - # resource name" that passes muster with VPC Service Control rules - # - # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 - # https://cloud.google.com/apis/design/resource_names#relative_resource_name - source_image_project_normalized = (can(var.instance_image.family) ? - "projects/${data.google_compute_image.slurm.project}/global/images/family" : - "projects/${data.google_compute_image.slurm.project}/global/images" - ) - source_image_family = can(var.instance_image.family) ? data.google_compute_image.slurm.family : "" - source_image = can(var.instance_image.name) ? data.google_compute_image.slurm.name : "" -} - -data "google_compute_image" "slurm" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = var.instance_image.project - - lifecycle { - precondition { - condition = length(regexall("^projects/.+?/global/images/family$", var.instance_image.project)) == 0 - error_message = "The \"project\" field in var.instance_image no longer supports a long-form ending in \"family\". Specify only the project ID." - } - - postcondition { - condition = var.instance_image_custom || contains(keys(local.known_project_families), self.project) - error_message = <<-EOD - Images in project ${self.project} are not published by SchedMD. Images must be created by compatible releases of the Terraform and Packer modules following the guidance at https://goo.gle/hpc-slurm-images. Set var.instance_image_custom to true to silence this error and acknowledge that you are using a compatible image. - EOD - } - postcondition { - condition = !contains(keys(local.known_project_families), self.project) || try(contains(local.known_project_families[self.project], self.family), false) - error_message = <<-EOD - Image family ${self.family} published by SchedMD in project ${self.project} is not compatible with this release of the Terraform Slurm modules. Select from known compatible releases: - ${join("\n", [for p in try(local.known_project_families[self.project], []) : "\t\"${p}\""])} - EOD - } - postcondition { - condition = var.disk_size_gb >= self.disk_size_gb - error_message = "'disk_size_gb: ${var.disk_size_gb}' is smaller than the image size (${self.disk_size_gb}GB), please increase the blueprint disk size" - } - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf deleted file mode 100644 index cd7a47f0a3..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/variables.tf +++ /dev/null @@ -1,809 +0,0 @@ -/** - * Copyright (C) SchedMD LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -########### -# GENERAL # -########### - -variable "project_id" { - type = string - description = "Project ID to create resources in." -} - -variable "deployment_name" { - description = "Name of the deployment." - type = string -} - -variable "slurm_cluster_name" { - type = string - description = <<-EOD - Cluster name, used for resource naming and slurm accounting. - If not provided it will default to the first 8 characters of the deployment name (removing any invalid characters). - EOD - default = null - - validation { - condition = var.slurm_cluster_name == null || can(regex("^[a-z](?:[a-z0-9]{0,9})$", var.slurm_cluster_name)) - error_message = "Variable 'slurm_cluster_name' must be a match of regex '^[a-z](?:[a-z0-9]{0,9})$'." - } -} - -variable "region" { - type = string - description = "The default region to place resources in." -} - -variable "zone" { - type = string - description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [gpu](#module\_gpu) | ../../../../modules/internal/gpu-definition | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_compute_image.slurm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_disks](#input\_additional\_disks) | List of maps of disks. | 
list(object({
    disk_name                  = string
    device_name                = string
    disk_type                  = string
    disk_size_gb               = number
    disk_labels                = map(string)
    auto_delete                = bool
    boot                       = bool
    disk_resource_manager_tags = map(string)
  }))
| `[]` | no | -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GCE, if any. | 
list(object({
    access_config = optional(list(object({
      nat_ip       = string
      network_tier = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier = string
    })), [])
    network            = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string)
    queue_count        = optional(number)
    stack_type         = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
  }))
| `[]` | no | -| [advanced\_machine\_features](#input\_advanced\_machine\_features) | See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features | 
object({
    enable_nested_virtualization = optional(bool)
    threads_per_core             = optional(number)
    turbo_mode                   = optional(string)
    visible_core_count           = optional(number)
    performance_monitoring_unit  = optional(string)
    enable_uefi_networking       = optional(bool)
  })
| 
{
  "threads_per_core": 1
}
| no | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Configures the network interface card and the maximum egress bandwidth for VMs.
- Setting `platform_default` respects the Google Cloud Platform API default values for networking.
- Setting `virtio_enabled` explicitly selects the VirtioNet network adapter.
- Setting `gvnic_enabled` selects the gVNIC network adapter (without Tier 1 high bandwidth).
- Setting `tier_1_enabled` selects both the gVNIC adapter and Tier 1 high bandwidth networking.
- Note: both gVNIC and Tier 1 networking require a VM image with gVNIC support as well as specific VM families and shapes.
- See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"platform_default"` | no | -| [can\_ip\_forward](#input\_can\_ip\_forward) | Enable IP forwarding, for NAT instances for example. | `bool` | `false` | no | -| [disable\_login\_public\_ips](#input\_disable\_login\_public\_ips) | DEPRECATED: Use `enable_login_public_ips` instead. | `bool` | `null` | no | -| [disable\_smt](#input\_disable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [disk\_auto\_delete](#input\_disk\_auto\_delete) | Whether or not the boot disk should be auto-deleted. | `bool` | `true` | no | -| [disk\_labels](#input\_disk\_labels) | Labels specific to the boot disk. These will be merged with var.labels. | `map(string)` | `{}` | no | -| [disk\_resource\_manager\_tags](#input\_disk\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB. | `number` | `50` | no | -| [disk\_type](#input\_disk\_type) | Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme. | `string` | `"pd-ssd"` | no | -| [enable\_confidential\_vm](#input\_enable\_confidential\_vm) | Enable the Confidential VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_login\_public\_ips](#input\_enable\_login\_public\_ips) | If set to true. The login node will have a random public IP assigned to it. | `bool` | `false` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enables Google Cloud os-login for user login and authentication for VMs.
See https://cloud.google.com/compute/docs/oslogin | `bool` | `true` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration. Note: the instance image must support option. | `bool` | `false` | no | -| [enable\_smt](#input\_enable\_smt) | DEPRECATED: Use `advanced_machine_features.threads_per_core` instead. | `bool` | `null` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [instance\_image](#input\_instance\_image) | Defines the image that will be used in the Slurm controller VM instance.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "slurm-gcp-6-9-hpc-rocky-linux-8",
  "project": "schedmd-slurm-public"
}
| no | -| [instance\_image\_custom](#input\_instance\_image\_custom) | A flag that designates that the user is aware that they are requesting
to use a custom and potentially incompatible image for this Slurm on
GCP module.

If the field is set to false, only the compatible families and project
names will be accepted. The deployment will fail with any other image
family or name. If set to true, no checks will be done.

See: https://goo.gle/hpc-slurm-images | `bool` | `false` | no | -| [instance\_template](#input\_instance\_template) | DEPRECATED: Instance template can not be specified for login nodes. | `string` | `null` | no | -| [labels](#input\_labels) | Labels, provided as a map. | `map(string)` | `{}` | no | -| [machine\_type](#input\_machine\_type) | Machine type to create. | `string` | `"c2-standard-4"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | -| [name\_prefix](#input\_name\_prefix) | Unique name prefix for login nodes. Automatically populated by the module id if not set.
If setting manually, ensure a unique value across all login groups. | `string` | n/a | yes | -| [num\_instances](#input\_num\_instances) | Number of instances to create. This value is ignored if static\_ips is provided. | `number` | `1` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy. | `string` | `"MIGRATE"` | no | -| [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | -| [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | -| [region](#input\_region) | Region where the instances should be created. | `string` | `null` | no | -| [resource\_manager\_tags](#input\_resource\_manager\_tags) | (Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag\_key\_id}, and values are in the format tagValues/456. | `map(string)` | `{}` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to attach to the login instances. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to attach to the login instances. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [static\_ips](#input\_static\_ips) | List of static IPs for VM instances. | `list(string)` | `[]` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Subnet to deploy to. | `string` | n/a | yes | -| [tags](#input\_tags) | Network tag list. | `list(string)` | `[]` | no | -| [zone](#input\_zone) | Zone where the instances should be created. If not specified, instances will be
spread across available zones in the region. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [login\_nodes](#output\_login\_nodes) | Slurm login instance definition. | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf deleted file mode 100644 index c12ee4627b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/main.tf +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "schedmd-slurm-gcp-v6-login", ghpc_role = "scheduler" }) -} - -module "gpu" { - source = "../../../../modules/internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - metadata = merge( - local.disable_automatic_updates_metadata, - var.metadata - ) - - additional_disks = [ - for ad in var.additional_disks : { - disk_name = ad.disk_name - device_name = ad.device_name - disk_type = ad.disk_type - disk_size_gb = ad.disk_size_gb - disk_labels = merge(ad.disk_labels, local.labels) - auto_delete = ad.auto_delete - boot = ad.boot - disk_resource_manager_tags = ad.var.disk_resource_manager_tags - } - ] - - public_access_config = [{ nat_ip = null, network_tier = null }] - - service_account = { - email = var.service_account_email - scopes = var.service_account_scopes - } - - # lower, replace `_` with `-`, and remove any non-alphanumeric characters - group_name = replace( - replace( - lower(var.name_prefix), - "_", "-"), - "/[^-a-z0-9]/", "") - - - login_node = { - group_name = local.group_name - disk_auto_delete = var.disk_auto_delete - disk_labels = merge(var.disk_labels, local.labels) - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - disk_resource_manager_tags = var.disk_resource_manager_tags - additional_disks = local.additional_disks - additional_networks = var.additional_networks - - can_ip_forward = var.can_ip_forward - advanced_machine_features = var.advanced_machine_features - - enable_confidential_vm = var.enable_confidential_vm - access_config = var.enable_login_public_ips ? local.public_access_config : [] - enable_oslogin = var.enable_oslogin - enable_shielded_vm = var.enable_shielded_vm - shielded_instance_config = var.shielded_instance_config - - gpu = one(local.guest_accelerator) - labels = local.labels - machine_type = var.machine_type - metadata = local.metadata - min_cpu_platform = var.min_cpu_platform - num_instances = var.num_instances - on_host_maintenance = var.on_host_maintenance - preemptible = var.preemptible - region = var.region - resource_manager_tags = var.resource_manager_tags - zone = var.zone - - service_account = local.service_account - - source_image_family = local.source_image_family # requires source_image_logic.tf - source_image_project = local.source_image_project_normalized # requires source_image_logic.tf - source_image = local.source_image # requires source_image_logic.tf - - static_ips = var.static_ips - bandwidth_tier = var.bandwidth_tier - - subnetwork = var.subnetwork_self_link - tags = var.tags - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml deleted file mode 100644 index 47f003258e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] -ghpc: - inject_module_id: name_prefix - has_to_be_used: true diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf deleted file mode 100644 index e700542794..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/outputs.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "login_nodes" { - description = "Slurm login instance definition." - value = [local.login_node] -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf deleted file mode 100644 index 0641cea7a8..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/source_image_logic.tf +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # Currently supported images and projects - known_project_families = { - schedmd-slurm-public = [ - "slurm-gcp-6-9-debian-11", - "slurm-gcp-6-9-hpc-rocky-linux-8", - "slurm-gcp-6-9-ubuntu-2004-lts", - "slurm-gcp-6-9-ubuntu-2204-lts-arm64" - ] - } - - # This approach to "hacking" the project name allows a chain of Terraform - # calls to set the instance source_image (boot disk) with a "relative - # resource name" that passes muster with VPC Service Control rules - # - # https://github.com/terraform-google-modules/terraform-google-vm/blob/735bd415fc5f034d46aa0de7922e8fada2327c0c/modules/instance_template/main.tf#L28 - # https://cloud.google.com/apis/design/resource_names#relative_resource_name - source_image_project_normalized = (can(var.instance_image.family) ? - "projects/${data.google_compute_image.slurm.project}/global/images/family" : - "projects/${data.google_compute_image.slurm.project}/global/images" - ) - source_image_family = can(var.instance_image.family) ? data.google_compute_image.slurm.family : "" - source_image = can(var.instance_image.name) ? data.google_compute_image.slurm.name : "" -} - -data "google_compute_image" "slurm" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = var.instance_image.project - - lifecycle { - precondition { - condition = length(regexall("^projects/.+?/global/images/family$", var.instance_image.project)) == 0 - error_message = "The \"project\" field in var.instance_image no longer supports a long-form ending in \"family\". Specify only the project ID." - } - - postcondition { - condition = var.instance_image_custom || contains(keys(local.known_project_families), self.project) - error_message = <<-EOD - Images in project ${self.project} are not published by SchedMD. Images must be created by compatible releases of the Terraform and Packer modules following the guidance at https://goo.gle/hpc-slurm-images. Set var.instance_image_custom to true to silence this error and acknowledge that you are using a compatible image. - EOD - } - postcondition { - condition = !contains(keys(local.known_project_families), self.project) || try(contains(local.known_project_families[self.project], self.family), false) - error_message = <<-EOD - Image family ${self.family} published by SchedMD in project ${self.project} is not compatible with this release of the Terraform Slurm modules. Select from known compatible releases: - ${join("\n", [for p in try(local.known_project_families[self.project], []) : "\t\"${p}\""])} - EOD - } - postcondition { - condition = var.disk_size_gb >= self.disk_size_gb - error_message = "'disk_size_gb: ${var.disk_size_gb}' is smaller than the image size (${self.disk_size_gb}GB), please increase the blueprint disk size" - } - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf deleted file mode 100644 index 8b51cf1ed2..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scheduler/schedmd-slurm-gcp-v6-login/variables.tf +++ /dev/null @@ -1,419 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -variable "project_id" { # tflint-ignore: terraform_unused_declarations - type = string - description = "Project ID to create resources in." -} - -variable "region" { - type = string - description = "Region where the instances should be created." - default = null -} - -variable "zone" { - type = string - description = <<-EOD - Zone where the instances should be created. If not specified, instances will be - spread across available zones in the region. - EOD - default = null -} - -variable "name_prefix" { - type = string - description = <<-EOD - Unique name prefix for login nodes. Automatically populated by the module id if not set. - If setting manually, ensure a unique value across all login groups. - EOD -} - -variable "num_instances" { - type = number - description = "Number of instances to create. This value is ignored if static_ips is provided." - default = 1 -} - -variable "resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instances. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} -} - -variable "disk_type" { - type = string - description = "Boot disk type, can be either hyperdisk-balanced, pd-ssd, pd-standard, pd-balanced, or pd-extreme." - default = "pd-ssd" -} - -variable "disk_size_gb" { - type = number - description = "Boot disk size in GB." - default = 50 -} - -variable "disk_auto_delete" { - type = bool - description = "Whether or not the boot disk should be auto-deleted." - default = true -} - -variable "disk_labels" { - description = "Labels specific to the boot disk. These will be merged with var.labels." - type = map(string) - default = {} -} - -variable "disk_resource_manager_tags" { - description = "(Optional) A set of key/value resource manager tag pairs to bind to the instance disks. Keys must be in the format tagKeys/{tag_key_id}, and values are in the format tagValues/456." - type = map(string) - default = {} - validation { - condition = alltrue([for value in var.disk_resource_manager_tags : can(regex("tagValues/[0-9]+", value))]) - error_message = "All Resource Manager tag values should be in the format 'tagValues/[0-9]+'" - } - validation { - condition = alltrue([for value in keys(var.disk_resource_manager_tags) : can(regex("tagKeys/[0-9]+", value))]) - error_message = "All Resource Manager tag keys should be in the format 'tagKeys/[0-9]+'" - } -} - -variable "additional_disks" { - type = list(object({ - disk_name = string - device_name = string - disk_type = string - disk_size_gb = number - disk_labels = map(string) - auto_delete = bool - boot = bool - disk_resource_manager_tags = map(string) - })) - description = "List of maps of disks." - default = [] -} - -variable "additional_networks" { - description = "Additional network interface details for GCE, if any." - default = [] - type = list(object({ - access_config = optional(list(object({ - nat_ip = string - network_tier = string - })), []) - alias_ip_range = optional(list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })), []) - ipv6_access_config = optional(list(object({ - network_tier = string - })), []) - network = optional(string) - network_ip = optional(string, "") - nic_type = optional(string) - queue_count = optional(number) - stack_type = optional(string) - subnetwork = optional(string) - subnetwork_project = optional(string) - })) - nullable = false -} - -variable "advanced_machine_features" { - description = "See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#nested_advanced_machine_features" - type = object({ - enable_nested_virtualization = optional(bool) - threads_per_core = optional(number) - turbo_mode = optional(string) - visible_core_count = optional(number) - performance_monitoring_unit = optional(string) - enable_uefi_networking = optional(bool) - }) - default = { - threads_per_core = 1 # disable SMT by default - } -} - -variable "enable_smt" { # tflint-ignore: terraform_unused_declarations - type = bool - description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - default = null - validation { - condition = var.enable_smt == null - error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - } -} - -variable "disable_smt" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - type = bool - default = null - validation { - condition = var.disable_smt == null - error_message = "DEPRECATED: Use `advanced_machine_features.threads_per_core` instead." - } -} - -variable "static_ips" { - type = list(string) - description = "List of static IPs for VM instances." - default = [] -} - -variable "bandwidth_tier" { - description = < -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13.0 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [condor\_version](#input\_condor\_version) | Yum/DNF-compatible version string; leave unset to use latest 23.0 LTS release (examples: "23.0.0","23.*")) | `string` | `"23.*"` | no | -| [enable\_docker](#input\_enable\_docker) | Install and enable docker daemon alongside HTCondor | `bool` | `true` | no | -| [http\_proxy](#input\_http\_proxy) | Set system default web (http and https) proxy for Windows HTCondor installation | `string` | `""` | no | -| [python\_windows\_installer\_url](#input\_python\_windows\_installer\_url) | URL of Python installer for Windows | `string` | `"https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [gcp\_service\_list](#output\_gcp\_service\_list) | Google Cloud APIs required by HTCondor | -| [runners](#output\_runners) | Runner to install HTCondor using startup-scripts | -| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | Windows PowerShell script to install HTCondor | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py deleted file mode 100644 index 77bafa0310..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/autoscaler.py +++ /dev/null @@ -1,417 +0,0 @@ -#!/usr/bin/python3 -# -*- coding: utf-8 -*- - -# Copyright 2018 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Script for resizing managed instance group (MIG) cluster size based -# on the number of jobs in the Condor Queue. - -from absl import app -from absl import flags -from collections import OrderedDict -from datetime import datetime -from pprint import pprint -from googleapiclient import discovery -from oauth2client.client import GoogleCredentials - -import argparse -import os -import math -import time -import htcondor -import classad - -parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) -parser.add_argument("--p", required=True, help="Project id", type=str) -parser.add_argument( - "--z", - required=True, - help="Name of GCP zone where the managed instance group is located", - type=str, -) -parser.add_argument( - "--r", - required=True, - help="Name of GCP region where the managed instance group is located", - type=str, -) -parser.add_argument( - "--mz", - required=False, - help="Enabled multizone (regional) managed instance group", - action="store_true", -) -parser.add_argument( - "--g", required=True, help="Name of the managed instance group", type=str -) -parser.add_argument( - "--i", - default=0, - help="Minimum number of idle compute instances", - type=int -) -parser.add_argument( - "--c", required=True, help="Maximum number of compute instances", type=int -) -parser.add_argument( - "--v", - default=0, - help="Increase output verbosity. 1-show basic debug info. 2-show detail debug info", - type=int, - choices=[0, 1, 2], -) -parser.add_argument( - "--d", - default=0, - help="Dry Run, default=0, if 1, then no scaling actions", - type=int, - choices=[0, 1], -) - -args = parser.parse_args() - -class AutoScaler: - def __init__(self, multizone=False): - - self.multizone = multizone - # Obtain credentials - self.credentials = GoogleCredentials.get_application_default() - self.service = discovery.build("compute", "v1", credentials=self.credentials) - - if self.multizone: - self.instanceGroupManagers = self.service.regionInstanceGroupManagers() - else: - self.instanceGroupManagers = self.service.instanceGroupManagers() - - # Remove specified instances from MIG and decrease MIG size - def deleteFromMig(self, node_self_links): - requestDelInstance = self.instanceGroupManagers.deleteInstances( - project=self.project, - **self.zoneargs, - instanceGroupManager=self.instance_group_manager, - body={ "instances": node_self_links }, - ) - - # execute if not a dry-run - if not self.dryrun: - response = requestDelInstance.execute() - if self.debug > 0: - pprint(response) - return response - return "Dry Run" - - def getInstanceTemplateInfo(self): - requestTemplateName = self.instanceGroupManagers.get( - project=self.project, - **self.zoneargs, - instanceGroupManager=self.instance_group_manager, - fields="instanceTemplate", - ) - responseTemplateName = requestTemplateName.execute() - template_name = "" - - if self.debug > 1: - print("Request for the template name") - pprint(responseTemplateName) - - if len(responseTemplateName) > 0: - template_url = responseTemplateName.get("instanceTemplate") - template_url_partitioned = template_url.split("/") - template_name = template_url_partitioned[len(template_url_partitioned) - 1] - - requestInstanceTemplate = self.service.instanceTemplates().get( - project=self.project, instanceTemplate=template_name, fields="properties" - ) - responseInstanceTemplateInfo = requestInstanceTemplate.execute() - - if self.debug > 1: - print("Template information") - pprint(responseInstanceTemplateInfo["properties"]) - - machine_type = responseInstanceTemplateInfo["properties"]["machineType"] - is_spot = responseInstanceTemplateInfo["properties"]["scheduling"][ - "preemptible" - ] - if self.debug > 0: - print("Machine Type: " + machine_type) - print("Is spot: " + str(is_spot)) - request = self.service.machineTypes().get( - project=self.project, zone=self.zone, machineType=machine_type - ) - response = request.execute() - guest_cpus = response["guestCpus"] - if self.debug > 1: - print("Machine information") - pprint(responseInstanceTemplateInfo["properties"]) - if self.debug > 0: - print("Guest CPUs: " + str(guest_cpus)) - - instanceTemplateInfo = { - "machine_type": machine_type, - "is_spot": is_spot, - "guest_cpus": guest_cpus, - } - return instanceTemplateInfo - - def scale(self): - # diagnosis - if self.debug > 1: - print("Launching autoscaler.py with the following arguments:") - print("project_id: " + self.project) - print("zone: " + self.zone) - print("region: " + self.region) - print(f"multizone: {self.multizone}") - print("group_manager: " + self.instance_group_manager) - print("computeinstancelimit: " + str(self.compute_instance_limit)) - print("debuglevel: " + str(self.debug)) - - if self.multizone: - self.zoneargs = {"region": self.region} - else: - self.zoneargs = {"zone": self.zone} - - # Each HTCondor scheduler (SchedD), maintains a list of jobs under its - # stewardship. A full list of Job ClassAd attributes can be found at - # https://htcondor.readthedocs.io/en/latest/classad-attributes/job-classad-attributes.html - schedd = htcondor.Schedd() - # encourage the job queue to start a new negotiation cycle; there are - # internal unconfigurable rate limits so not guaranteed; this is not - # strictly required for success, but may reduce latency of autoscaling - schedd.reschedule() - REQUEST_CPUS_ATTRIBUTE = "RequestCpus" - REQUEST_GPUS_ATTRIBUTE = "RequestGpus" - REQUEST_MEMORY_ATTRIBUTE = "RequestMemory" - job_attributes = [ - REQUEST_CPUS_ATTRIBUTE, - REQUEST_GPUS_ATTRIBUTE, - REQUEST_MEMORY_ATTRIBUTE, - ] - - instanceTemplateInfo = self.getInstanceTemplateInfo() - self.is_spot = instanceTemplateInfo["is_spot"] - self.cores_per_node = instanceTemplateInfo["guest_cpus"] - print(f"MIG is configured for Spot pricing: {self.is_spot}") - print("Number of CPU per compute node: " + str(self.cores_per_node)) - - # this query will constrain the search for jobs to those that either - # require spot VMs or do not require Spot VMs based on whether the - # VM instance template is configured for Spot pricing - spot_query = classad.ExprTree(f"RequireId == \"{self.instance_group_manager}\"") - - # For purpose of scaling a Managed Instance Group, count only jobs that - # are idle and likely participated in a negotiation cycle (there does - # not appear to be a single classad attribute for this). - # https://htcondor.readthedocs.io/en/latest/classad-attributes/job-classad-attributes.html#JobStatus - LAST_CYCLE_ATTRIBUTE = "LastNegotiationCycleTime0" - coll = htcondor.Collector() - negotiator_ad = coll.query(htcondor.AdTypes.Negotiator, projection=[LAST_CYCLE_ATTRIBUTE]) - if len(negotiator_ad) != 1: - print(f"There should be exactly 1 negotiator in the pool. There is {len(negotiator_ad)}") - exit() - last_negotiation_cycle_time = negotiator_ad[0].get(LAST_CYCLE_ATTRIBUTE) - if not last_negotiation_cycle_time: - print(f"The negotiator has not yet started a match cycle. Exiting auto-scaling.") - exit() - - print(f"Last negotiation cycle occurred at: {datetime.fromtimestamp(last_negotiation_cycle_time)}") - idle_job_query = classad.ExprTree(f"JobStatus == 1 && QDate < {last_negotiation_cycle_time}") - idle_job_ads = schedd.query(constraint=idle_job_query.and_(spot_query), - projection=job_attributes) - - total_idle_request_cpus = sum(j[REQUEST_CPUS_ATTRIBUTE] for j in idle_job_ads) - print(f"Total CPUs requested by idle jobs: {total_idle_request_cpus}") - - if self.debug > 1: - print("Information about the compute instance template") - pprint(instanceTemplateInfo) - - # Calculate the minimum number of instances that, for fully packed - # execute points, could satisfy current job queue - min_hosts_for_idle_jobs = math.ceil(total_idle_request_cpus / self.cores_per_node) - if self.debug > 0: - print(f"Minimum hosts needed: {total_idle_request_cpus} / {self.cores_per_node} = {min_hosts_for_idle_jobs}") - - # Get current number of instances in the MIG - requestGroupInfo = self.instanceGroupManagers.get( - project=self.project, - **self.zoneargs, - instanceGroupManager=self.instance_group_manager, - ) - responseGroupInfo = requestGroupInfo.execute() - current_target = responseGroupInfo["targetSize"] - print(f"Current MIG target size: {current_target}") - - # Find instances that are being modified by the MIG (currentAction is - # any value other than "NONE"). A common reason an instance is modified - # is it because it has failed a health check. - reqModifyingInstances = self.instanceGroupManagers.listManagedInstances( - project=self.project, - **self.zoneargs, - instanceGroupManager=self.instance_group_manager, - filter="currentAction != \"NONE\"", - orderBy="creationTimestamp desc" - ) - respModifyingInstances = reqModifyingInstances.execute() - - # Find VMs that are idle (no dynamic slots created from partitionable - # slots) in the MIG handled by this autoscaler - filter_idle_vms = classad.ExprTree(f"PartitionableSlot && NumDynamicSlots==0") - filter_claimed_vms = classad.ExprTree(f"PartitionableSlot && NumDynamicSlots>0") - filter_mig = classad.ExprTree(f"regexp(\".*/{self.instance_group_manager}$\", CloudCreatedBy)") - # A full list of Machine (StartD) ClassAd attributes can be found at - # https://htcondor.readthedocs.io/en/latest/classad-attributes/machine-classad-attributes.html - idle_node_ads = coll.query(htcondor.AdTypes.Startd, - constraint=filter_idle_vms.and_(filter_mig), - projection=["Machine", "CloudZone"]) - - NODENAME_ATTRIBUTE = "Machine" - claimed_node_ads = coll.query(htcondor.AdTypes.Startd, - constraint=filter_claimed_vms.and_(filter_mig), - projection=[NODENAME_ATTRIBUTE]) - claimed_nodes = [ ad[NODENAME_ATTRIBUTE].split(".")[0] for ad in claimed_node_ads] - - # treat OrderedDict as a set by ignoring key values; this set will - # contain VMs we would consider deleting, in inverse order of - # their readiness to join pool (creating, unhealthy, healthy+idle) - idle_nodes = OrderedDict() - try: - modifyingInstances = respModifyingInstances["managedInstances"] - except KeyError: - modifyingInstances = [] - - print(f"There are {len(modifyingInstances)} VMs being modified by the managed instance group") - - # there is potential for nodes in MIG health check "VERIFYING" state - # to have already joined the pool and be running jobs - for instance in modifyingInstances: - self_link = instance["instance"] - node_name = self_link.rsplit("/", 1)[-1] - if node_name not in claimed_nodes: - idle_nodes[self_link] = "modifying" - - for ad in idle_node_ads: - node = ad["Machine"].split(".")[0] - zone = ad["CloudZone"] - self_link = "https://www.googleapis.com/compute/v1/projects/" + \ - self.project + "/zones/" + zone + "/instances/" + node - # there is potential for nodes in MIG health check "VERIFYING" state - # to have already joined the pool and be idle; delete them last - if self_link in idle_nodes: - idle_nodes.move_to_end(self_link) - idle_nodes[self_link] = "idle" - n_idle = len(idle_nodes) - - print(f"There are {n_idle} VMs being modified or idle in the pool") - if self.debug > 1: - print("Listing idle nodes:") - pprint(idle_nodes) - - # always keep size tending toward the minimum idle VMs requested - new_target = current_target + self.compute_instance_min_idle - n_idle + min_hosts_for_idle_jobs - if new_target > self.compute_instance_limit: - self.size = self.compute_instance_limit - print(f"MIG target size will be limited by {self.compute_instance_limit}") - else: - self.size = new_target - - print(f"New MIG target size: {self.size}") - - if self.debug > 1: - print("MIG Information:") - print(responseGroupInfo) - - if self.size == current_target: - if current_target == 0: - print("Queue is empty") - print("Running correct number of VMs to handle queue") - exit() - - if self.size < current_target: - print("Scaling down. Looking for nodes that can be shut down") - - if self.debug > 1: - print("Compute node busy status:") - for node in idle_nodes: - print(node) - - # Shut down idle nodes up to our calculated limit - nodes_to_delete = list(idle_nodes.keys())[0:current_target-self.size] - for node in nodes_to_delete: - print(f"Attempting to delete: {node.rsplit('/',1)[-1]}") - respDel = self.deleteFromMig(nodes_to_delete) - - if self.debug > 1: - print("Scaling down complete") - - if self.size > current_target: - print( - "Scaling up. Need to increase number of instances to " + str(self.size) - ) - # Request to resize - request = self.instanceGroupManagers.resize( - project=self.project, - **self.zoneargs, - instanceGroupManager=self.instance_group_manager, - size=self.size, - ) - response = request.execute() - if self.debug > 1: - print("Requesting to increase MIG size") - pprint(response) - print("Scaling up complete") - - -def main(): - - scaler = AutoScaler(args.mz) - - # Project ID - scaler.project = args.p # Ex:'slurm-var-demo' - - # Name of the zone where the managed instance group is located - scaler.zone = args.z # Ex: 'us-central1-f' - - # Name of the region where the managed instance group is located - scaler.region = args.r # Ex: 'us-central1' - - # The name of the managed instance group. - scaler.instance_group_manager = args.g # Ex: 'condor-compute-igm' - - # Default number of cores per instance, will be replaced with actual value - scaler.cores_per_node = 4 - - # Default number of running instances that the managed instance group should maintain at any given time. This number will go up and down based on the load (number of jobs in the queue) - scaler.size = 0 - - scaler.compute_instance_min_idle = args.i - - # Dry run: : 0, run scaling; 1, only provide info. - scaler.dryrun = args.d > 0 - - # Debug level: 1-print debug information, 2 - print detail debug information - scaler.debug = 0 - if args.v: - scaler.debug = args.v - - # Limit for the maximum number of compute instance. If zero (default setting), no limit will be enforced by the script - scaler.compute_instance_limit = 0 - if args.c: - scaler.compute_instance_limit = abs(args.c) - - scaler.scale() - - -if __name__ == "__main__": - main() diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml deleted file mode 100644 index db989f9d40..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor-autoscaler-deps.yml +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Install but do not activate HTCondor autoscaler - become: true - hosts: localhost - tasks: - - name: Install Python 3 pip - ansible.builtin.package: - name: python3-pip - state: present - - name: Create virtual environment for HTCondor autoscaler - ansible.builtin.pip: - name: pip - version: 21.3.1 # last Python 3.6-compatible release - virtualenv: /usr/local/htcondor - virtualenv_command: /usr/bin/python3 -m venv - - name: Install latest setuptools - ansible.builtin.pip: - name: setuptools - version: 59.6.0 # last Python 3.6-compatible release - virtualenv: /usr/local/htcondor - virtualenv_command: /usr/bin/python3 -m venv - - name: Install HTCondor autoscaler dependencies - with_items: - - oauth2client - - google-api-python-client - - absl-py - - htcondor - ansible.builtin.pip: - name: "{{ item }}" - state: present # rely on pip resolver to pick latest compatible releases - virtualenv: /usr/local/htcondor - virtualenv_command: /usr/bin/python3 -m venv diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml deleted file mode 100644 index 4d3abbbfd6..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/files/install-htcondor.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# The instructions for installing HTCondor may change with time, although we -# anticipate that they will stay fixed for the 23.0 releases. Find up-to-date -# recommendations at: -## https://htcondor.readthedocs.io/en/latest/getting-htcondor/from-our-repositories.html - ---- -- name: Ensure HTCondor is installed - hosts: all - vars: - enable_docker: true - htcondor_key: https://research.cs.wisc.edu/htcondor/repo/keys/HTCondor-23.0-Key - docker_key: https://download.docker.com/linux/centos/gpg - become: true - module_defaults: - ansible.builtin.yum: - lock_timeout: 300 - tasks: - - name: Enable EPEL repository - ansible.builtin.yum: - name: - - epel-release - - name: Directly install RPM verification keys - ansible.builtin.rpm_key: - state: present - key: "{{ item }}" - loop: - - "{{ htcondor_key }}" - - "{{ docker_key }}" - register: key_install - retries: 10 - delay: 60 - until: key_install is success - - name: Enable HTCondor LTS Release repository - ansible.builtin.yum_repository: - name: htcondor-feature - description: HTCondor LTS Release (23.0) - file: htcondor - baseurl: https://research.cs.wisc.edu/htcondor/repo/23.0/el$releasever/$basearch/release - gpgkey: "{{ htcondor_key }}" - gpgcheck: true - repo_gpgcheck: true - priority: "90" - - name: Install HTCondor - ansible.builtin.yum: - name: condor-{{ condor_version | default("23.*") | string }} - state: present - - name: Ensure token directory - ansible.builtin.file: - path: /etc/condor/tokens.d - mode: 0700 - owner: root - group: root - - name: Install Docker and configure HTCondor to use it - when: enable_docker | bool # allows string to be passed at CLI - block: - - name: Setup Docker repo - ansible.builtin.yum_repository: - name: docker-ce-stable - description: Docker CE Stable - $basearch - baseurl: https://download.docker.com/linux/centos/$releasever/$basearch/stable - enabled: yes - gpgcheck: yes - gpgkey: "{{ docker_key }}" - - name: Install Docker - ansible.builtin.yum: - name: - - docker-ce - - docker-ce-cli - - containerd.io - - docker-compose-plugin - - name: Enable Docker - ansible.builtin.service: - name: docker - state: started - enabled: true - - name: Add condor to docker group - ansible.builtin.user: - name: condor - groups: docker - append: yes diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/main.tf deleted file mode 100644 index 0853e035f4..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/main.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - runners = [ - { - "type" = "ansible-local" - "source" = "${path.module}/files/install-htcondor.yaml" - "destination" = "install-htcondor.yaml" - "args" = join(" ", [ - "-e enable_docker=${var.enable_docker}", - "-e condor_version=${var.condor_version}", - ]) - }, - { - "type" = "ansible-local" - "content" = file("${path.module}/files/install-htcondor-autoscaler-deps.yml") - "destination" = "install-htcondor-autoscaler-deps.yml" - }, - { - "type" = "data" - "content" = file("${path.module}/files/autoscaler.py") - "destination" = "/usr/local/htcondor/bin/autoscaler.py" - }, - ] - - install_htcondor_ps1 = templatefile( - "${path.module}/templates/install-htcondor.ps1.tftpl", { - condor_version = var.condor_version, - http_proxy = var.http_proxy, - python_windows_installer_url = var.python_windows_installer_url, - }) - - required_apis = [ - "compute.googleapis.com", - "secretmanager.googleapis.com", - ] -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf deleted file mode 100644 index c7951737ff..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "runners" { - description = "Runner to install HTCondor using startup-scripts" - value = local.runners -} - -output "windows_startup_ps1" { - description = "Windows PowerShell script to install HTCondor" - value = local.install_htcondor_ps1 -} - -output "gcp_service_list" { - description = "Google Cloud APIs required by HTCondor" - value = local.required_apis -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl deleted file mode 100644 index 7492da3c12..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/templates/install-htcondor.ps1.tftpl +++ /dev/null @@ -1,59 +0,0 @@ -#Requires -RunAsAdministrator - -# Windows 2016 needs forced upgrade to TLS 1.2 -[Net.ServicePointManager]::SecurityProtocol = 'Tls12' - -# important for catching exception in Invoke-WebRequest -Set-StrictMode -Version latest -$ErrorActionPreference = 'Stop' - -%{ if http_proxy != "" ~} -[System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy("${http_proxy}") -%{ endif ~} - -# do not show progress bar when running Invoke-WebRequest -$ProgressPreference = 'SilentlyContinue' - -# download C Runtime DLL necessary for HTCondor installer -$runtime_installer = 'C:\vc_redist.x64.exe' -Invoke-WebRequest https://aka.ms/vs/17/release/vc_redist.x64.exe -OutFile "$runtime_installer" -Start-Process -FilePath "$runtime_installer" -Wait -ArgumentList "/norestart /quiet /log c:\vc_redist_log.txt" -Remove-Item "$runtime_installer" - -# download HTCondor installer -$htcondor_installer = 'C:\htcondor.msi' -%{ if condor_version == "23.*" } -Invoke-WebRequest https://research.cs.wisc.edu/htcondor/tarball/23.0/current/condor-Windows-x64.msi -OutFile "$htcondor_installer" -%{ else ~} -Invoke-WebRequest https://research.cs.wisc.edu/htcondor/tarball/23.0/${condor_version}/release/condor-${condor_version}-Windows-x64.msi -OutFile "$htcondor_installer" -%{ endif ~} -$args='/qn /l* condor-install-log.txt /i' -$args=$args + " $htcondor_installer" -$args=$args + ' NEWPOOL="N"' -$args=$args + ' RUNJOBS="N"' -$args=$args + ' SUBMITJOBS="N"' -$args=$args + ' INSTALLDIR="C:\Condor"' -Start-Process "msiexec.exe" -Wait -ArgumentList "$args" -Remove-Item "$htcondor_installer" - -# do not start HTCondor on boot by default. Allow startup script to download -# configuration first and then start HTCondor -Set-Service -StartupType Manual condor - -# remove settings from condor_config that we want to override in configuration step -Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^CONDOR_HOST' -NotMatch) -Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^INSTALL_USER' -NotMatch) -Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^DAEMON_LIST' -NotMatch) -Set-Content -Path "C:\Condor\condor_config" -Value (Get-Content -Path "C:\Condor\condor_config" | Select-String -Pattern '^use SECURITY' -NotMatch) - -# install Python so that custom ClassAd hooks can execute -$python_installer = 'C:\python-installer.exe' -Invoke-WebRequest -Uri "${python_windows_installer_url}" -OutFile "$python_installer" -Start-Process -FilePath "$python_installer" -Wait -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1 Include_test=0' -%{ if http_proxy == "" ~} -Start-Process "py.exe" -Wait -ArgumentList "-3.11 -m pip install --no-warn-script-location requests" -%{ else ~} -Start-Process "py.exe" -Wait -ArgumentList "-3.11 -m pip install --proxy ${http_proxy} --no-warn-script-location requests" -%{ endif ~} -Invoke-WebRequest -Uri "https://raw.githubusercontent.com/htcondor/htcondor/main/src/condor_scripts/common-cloud-attributes-google.py" -OutFile "C:\Condor\bin\common-cloud-attributes-google.py" -Remove-Item "$python_installer" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/variables.tf deleted file mode 100644 index 1afdf4e0eb..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/variables.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "enable_docker" { - description = "Install and enable docker daemon alongside HTCondor" - type = bool - default = true -} - -variable "condor_version" { - description = "Yum/DNF-compatible version string; leave unset to use latest 23.0 LTS release (examples: \"23.0.0\",\"23.*\"))" - type = string - default = "23.*" - - validation { - error_message = "var.condor_version must be set to \"23.*\" for latest 23.0 release or to a specific \"23.0.y\" release." - condition = var.condor_version == "23.*" || ( - length(split(".", var.condor_version)) == 3 && alltrue([ - for v in split(".", var.condor_version) : can(tonumber(v)) - ]) && split(".", var.condor_version)[0] == "23" - && split(".", var.condor_version)[1] == "0" - ) - } -} - -variable "http_proxy" { - description = "Set system default web (http and https) proxy for Windows HTCondor installation" - type = string - default = "" - nullable = false -} - -variable "python_windows_installer_url" { - description = "URL of Python installer for Windows" - type = string - default = "https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe" - nullable = false -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/versions.tf deleted file mode 100644 index 79b6fbde47..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/htcondor-install/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.13.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/README.md deleted file mode 100644 index 55c2fc7e4e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/README.md +++ /dev/null @@ -1,116 +0,0 @@ -## Description - -This module will create a startup-script runner that will execute Ramble commands. - -Ramble is a multi-platform experimentation framework capable of driving -software installation, acquiring input files, configuring experiments, and -extracting results. For more information about Ramble, see: -https://github.com/GoogleCloudPlatform/ramble - -This module outputs a startup script runner, which can be combined with other -startup script runners to execute a set of Ramble commands. - -Ramble makes extensive use of Spack. It must be installed with a Toolkit runner -generated by the [spack-setup module](../spack-setup/README.md) following the -[basic example](#basic-example) below. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -# Examples - -## Basic Example - -Below is a basic example of using this module. - -```yaml - - id: spack - source: community/modules/scripts/spack-setup - - - id: ramble-setup - source: community/modules/scripts/ramble-setup - - - id: ramble-execute - source: community/modules/scripts/ramble-execute - use: [spack, ramble-setup] - settings: - commands: - - ramble list -``` - -This example shows installing Spack and Ramble with their own modules -(spack-setup and ramble-setup respectively). Then the ramble-execute module -is added to simply list all applications Ramble knows about. - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [local](#requirement\_local) | >= 2.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [local](#provider\_local) | >= 2.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [local_file.debug_file_ansible_execute](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [commands](#input\_commands) | String of commands to run within this module | `string` | `null` | no | -| [data\_files](#input\_data\_files) | A list of files to be transferred prior to running commands.
It must specify one of 'source' (absolute local file path) or 'content' (string).
It must specify a 'destination' with absolute path where file should be placed. | `list(map(string))` | `[]` | no | -| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing spack scripts. | `string` | n/a | yes | -| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | n/a | yes | -| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | -| [log\_file](#input\_log\_file) | Log file to write output from Ramble execute steps into | `string` | `"/var/log/ramble-execute.log"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [ramble\_profile\_script\_path](#input\_ramble\_profile\_script\_path) | Path to the Ramble profile.d script. Created by an instance of ramble-setup.
Can be defined explicitly, or by chaining an instance of a ramble-setup module
through a `use` setting. | `string` | n/a | yes | -| [ramble\_runner](#input\_ramble\_runner) | Runner from previous ramble-setup or ramble-execute to be chained with scripts generated by this module. | 
object({
    type        = string
    content     = string
    destination = string
  })
| n/a | yes | -| [region](#input\_region) | Region to place bucket containing spack scripts. | `string` | n/a | yes | -| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script.
Can be defined explicitly, or by chaining an instance of a spack-setup module
through a `use` setting.
Defaults to /etc/profile.d/spack.sh if not set. | `string` | `"/etc/profile.d/spack.sh"` | no | -| [system\_user\_name](#input\_system\_user\_name) | Name of the system user used to execute commands. Generally passed from the ramble-setup module. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [controller\_startup\_script](#output\_controller\_startup\_script) | Ramble startup script, duplicate for SLURM controller. | -| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for ramble, to be reused by ramble-execute module. | -| [ramble\_profile\_script\_path](#output\_ramble\_profile\_script\_path) | Path to Ramble profile script. | -| [ramble\_runner](#output\_ramble\_runner) | Runner to execute Ramble commands using an ansible playbook. The startup-script module
will automatically handle installation of ansible. | -| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to Spack profile script. | -| [startup\_script](#output\_startup\_script) | Ramble startup script. | -| [system\_user\_name](#output\_system\_user\_name) | The system user used to execute commands. | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/main.tf deleted file mode 100644 index 7ef0b029e3..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/main.tf +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "ramble-execute", ghpc_role = "scripts" }) -} - -locals { - commands_content = var.commands == null ? "echo 'no ramble commands provided'" : indent(4, yamlencode(var.commands)) - - execute_contents = templatefile( - "${path.module}/templates/ramble_execute.yml.tpl", - { - pre_script = "if [ -f ${var.spack_profile_script_path} ]; then . ${var.spack_profile_script_path}; fi; . ${var.ramble_profile_script_path}" - log_file = var.log_file - commands = local.commands_content - system_user_name = var.system_user_name - } - ) - - data_runners = [for data_file in var.data_files : merge(data_file, { type = "data" })] - - execute_md5 = substr(md5(local.execute_contents), 0, 4) - execute_runner = { - type = "ansible-local" - content = local.execute_contents - destination = "ramble_execute_${local.execute_md5}.yml" - } - - previous_runners = var.ramble_runner != null ? [var.ramble_runner] : [] - runners = concat(local.previous_runners, local.data_runners, [local.execute_runner]) - - # Destinations should be unique while also being known at time of apply - combined_unique_string = join("\n", [for runner in local.runners : runner["destination"]]) - combined_md5 = substr(md5(local.combined_unique_string), 0, 4) - combined_runner = { - type = "shell" - content = module.startup_script.startup_script - destination = "combined_install_ramble_${local.combined_md5}.sh" - } -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.runners - gcs_bucket_path = var.gcs_bucket_path -} - -resource "local_file" "debug_file_ansible_execute" { - content = local.execute_contents - filename = "${path.module}/debug_execute_${local.execute_md5}.yml" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf deleted file mode 100644 index 4e6c3a44d8..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/outputs.tf +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "Ramble startup script." - value = module.startup_script.startup_script -} - -output "controller_startup_script" { - description = "Ramble startup script, duplicate for SLURM controller." - value = module.startup_script.startup_script -} - -output "ramble_runner" { - description = <<-EOT - Runner to execute Ramble commands using an ansible playbook. The startup-script module - will automatically handle installation of ansible. - EOT - value = local.combined_runner -} - -output "gcs_bucket_path" { - description = "Bucket containing the startup scripts for ramble, to be reused by ramble-execute module." - value = var.gcs_bucket_path -} - -output "spack_profile_script_path" { - description = "Path to Spack profile script." - value = var.spack_profile_script_path -} - -output "ramble_profile_script_path" { - description = "Path to Ramble profile script." - value = var.ramble_profile_script_path -} - -output "system_user_name" { - description = "The system user used to execute commands." - value = var.system_user_name -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl deleted file mode 100644 index 0e98f3aa2c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/templates/ramble_execute.yml.tpl +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Execute Commands - hosts: localhost - vars: - pre_script: ${pre_script} - log_file: ${log_file} - commands: ${commands} - system_user_name: ${system_user_name} - tasks: - - name: Execute command block - block: - - name: Print commands to be executed - ansible.builtin.debug: - msg: "{{ commands.split('\n') | ansible.builtin.to_nice_yaml }}" - - - name: Streaming log info - ansible.builtin.debug: - msg: | - Logs from commands will not be printed here until success (or failure) - Streaming logs can be found at {{ log_file }} - - - name: Ensure user can write to log file - ansible.builtin.file: - path: "{{ log_file }}" - state: touch - owner: "{{ system_user_name }}" - - - name: Execute commands - ansible.builtin.shell: | - set -eo pipefail - { - {{ pre_script }} - echo " === Starting commands ===" - {{ commands }} - echo " === Finished commands ===" - } 2>&1 | tee -a {{ log_file }} - args: - executable: /bin/bash - register: output - become: true - become_user: "{{ system_user_name }}" - - always: - - name: Print commands output - ansible.builtin.debug: - var: output.stdout_lines diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/variables.tf deleted file mode 100644 index ec67228df5..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/variables.tf +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created." - type = string -} - -variable "deployment_name" { - description = "Name of deployment, used to name bucket containing spack scripts." - type = string -} - -variable "region" { - description = "Region to place bucket containing spack scripts." - type = string -} - -variable "labels" { - description = "Key-value pairs of labels to be added to created resources." - type = map(string) -} - -variable "log_file" { - description = "Log file to write output from Ramble execute steps into" - default = "/var/log/ramble-execute.log" - type = string -} - -variable "data_files" { - description = <<-EOT - A list of files to be transferred prior to running commands. - It must specify one of 'source' (absolute local file path) or 'content' (string). - It must specify a 'destination' with absolute path where file should be placed. - EOT - type = list(map(string)) - default = [] - validation { - condition = alltrue([for r in var.data_files : substr(r["destination"], 0, 1) == "/"]) - error_message = "All destinations must be absolute paths and start with '/'." - } - validation { - condition = alltrue([ - for r in var.data_files : - can(r["content"]) != can(r["source"]) - ]) - error_message = "A data_file must specify either 'content' or 'source', but never both." - } - validation { - condition = alltrue([ - for r in var.data_files : - lookup(r, "content", lookup(r, "source", null)) != null - ]) - error_message = "A data_file must specify a non-null 'content' or 'source'." - } -} - -variable "commands" { - description = "String of commands to run within this module" - default = null - type = string -} - -variable "ramble_runner" { - description = "Runner from previous ramble-setup or ramble-execute to be chained with scripts generated by this module." - type = object({ - type = string - content = string - destination = string - }) -} - -variable "system_user_name" { - description = "Name of the system user used to execute commands. Generally passed from the ramble-setup module." - type = string -} - -variable "gcs_bucket_path" { - description = "The GCS path for storage bucket and the object, starting with `gs://`." - type = string -} - -variable "spack_profile_script_path" { - description = <<-EOT - Path to the Spack profile.d script. - Can be defined explicitly, or by chaining an instance of a spack-setup module - through a `use` setting. - Defaults to /etc/profile.d/spack.sh if not set. - EOT - type = string - default = "/etc/profile.d/spack.sh" -} - -variable "ramble_profile_script_path" { - description = <<-EOT - Path to the Ramble profile.d script. Created by an instance of ramble-setup. - Can be defined explicitly, or by chaining an instance of a ramble-setup module - through a `use` setting. - EOT - type = string -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/versions.tf deleted file mode 100644 index 9b23317323..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-execute/versions.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0.0" - required_providers { - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/README.md deleted file mode 100644 index 9891088105..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/README.md +++ /dev/null @@ -1,128 +0,0 @@ -## Description - -This module will create a set of startup-script runners that will setup Ramble, -and install Rambleā€™s dependencies. - -Ramble is a multi-platform experimentation framework capable of driving -software installation, acquiring input files, configuring experiments, and -extracting results. For more information about ramble, see: -https://github.com/GoogleCloudPlatform/ramble - -This module outputs two startup script runners, which can be added to startup -scripts to setup, ramble and its dependencies. - -For this module to be completely functional, it depends on a spack -installation. For more information, see Cluster-Toolkitā€™s Spack module. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -# Examples - -## Basic Example - -```yaml -- id: ramble-setup - source: community/modules/scripts/ramble-setup -``` - -This example simply installs ramble on a VM. - -## Full Example - -```yaml -- id: ramble-setup - source: community/modules/scripts/ramble-setup - settings: - install_dir: /ramble - ramble_url: https://github.com/GoogleCloudPlatform/ramble - ramble_ref: v0.2.1 - log_file: /var/log/ramble.log - chown_owner: ā€œownerā€ - chgrp_group: ā€œuser_groupā€ - chmod_mode: ā€œa+rā€ -``` - -This example simply installs ramble into a VM at the location `/ramble`, checks -out the v0.2.1 tag, changes the owner and group to ā€œownerā€ and ā€œuser_groupā€, -and chmodā€™s the clone to make it world readable. - -Also see a more complete [Ramble example blueprint](../../../examples/ramble.yaml). - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [google](#requirement\_google) | >= 4.42 | -| [local](#requirement\_local) | >= 2.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [local](#provider\_local) | >= 2.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | -| [local_file.debug_file_shell_install](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [chmod\_mode](#input\_chmod\_mode) | Mode to chmod the Ramble clone to. Defaults to `""` (i.e. do not modify).
For usage information see:
https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode | `string` | `""` | no | -| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing startup script. | `string` | n/a | yes | -| [install\_dir](#input\_install\_dir) | Destination directory of installation of Ramble. | `string` | `"/apps/ramble"` | no | -| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [ramble\_profile\_script\_path](#input\_ramble\_profile\_script\_path) | Path to the Ramble profile.d script. Created by this module | `string` | `"/etc/profile.d/ramble.sh"` | no | -| [ramble\_ref](#input\_ramble\_ref) | Git ref to checkout for Ramble. | `string` | `"develop"` | no | -| [ramble\_url](#input\_ramble\_url) | URL for Ramble repository to clone. | `string` | `"https://github.com/GoogleCloudPlatform/ramble"` | no | -| [ramble\_virtualenv\_path](#input\_ramble\_virtualenv\_path) | Virtual environment path in which to install Ramble Python interpreter and other dependencies | `string` | `"/usr/local/ramble-python"` | no | -| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | -| [system\_user\_gid](#input\_system\_user\_gid) | GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary. | `number` | `1104762904` | no | -| [system\_user\_name](#input\_system\_user\_name) | Name of system user that will perform installation of Ramble. It will be created if it does not exist. | `string` | `"ramble"` | no | -| [system\_user\_uid](#input\_system\_user\_uid) | UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary. | `number` | `1104762904` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [controller\_startup\_script](#output\_controller\_startup\_script) | Ramble installation script, duplicate for SLURM controller. | -| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for Ramble, to be reused by ramble-execute module. | -| [ramble\_path](#output\_ramble\_path) | Location ramble is installed into. | -| [ramble\_profile\_script\_path](#output\_ramble\_profile\_script\_path) | Path to Ramble profile script. | -| [ramble\_ref](#output\_ramble\_ref) | Git ref the ramble install is checked out to use | -| [ramble\_runner](#output\_ramble\_runner) | Runner to be used with startup-script module or passed to ramble-execute module.
- installs Ramble dependencies
- installs Ramble
- generates profile.d script to enable access to Ramble
This is safe to run in parallel by multiple machines. | -| [startup\_script](#output\_startup\_script) | Ramble installation script. | -| [system\_user\_name](#output\_system\_user\_name) | The system user used to install Ramble. It can be reused by ramble-execute module to execute Ramble commands. | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/main.tf deleted file mode 100644 index 4389af7d33..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/main.tf +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "ramble-setup", ghpc_role = "scripts" }) -} - -locals { - profile_script = <<-EOF - if [ -f ${var.install_dir}/share/ramble/setup-env.sh ]; then - test -t 1 && echo "** Ramble's python virtualenv (/usr/local/ramble-python) is activated. Call 'deactivate' to deactivate." - VIRTUAL_ENV_DISABLE_PROMPT=1 . ${var.ramble_virtualenv_path}/bin/activate - . ${var.install_dir}/share/ramble/setup-env.sh - fi - EOF - - script_content = templatefile( - "${path.module}/templates/ramble_setup.yml.tftpl", - { - sw_name = "ramble" - profile_script = indent(4, yamlencode(local.profile_script)) - install_dir = var.install_dir - git_url = var.ramble_url - git_ref = var.ramble_ref - chmod_mode = var.chmod_mode - system_user_name = var.system_user_name - system_user_uid = var.system_user_uid - system_user_gid = var.system_user_gid - finalize_setup_script = "echo 'no finalize setup script'" - profile_script_path = var.ramble_profile_script_path - } - ) - - install_ramble_deps_runner = { - "type" = "ansible-local" - "source" = "${path.module}/scripts/install_ramble_deps.yml" - "destination" = "install_ramble_deps.yml" - "args" = "-e virtualenv_path=${var.ramble_virtualenv_path}" - } - - python_reqs_content = templatefile( - "${path.module}/templates/install_ramble_python_deps.yml.tftpl", - { - install_dir = var.install_dir - virtualenv_path = var.ramble_virtualenv_path - } - ) - - python_reqs_runner = { - "type" = "ansible-local" - "content" = local.python_reqs_content - "destination" = "install_ramble_reqs.yml" - } - - install_ramble_runner = { - "type" = "ansible-local" - "content" = local.script_content - "destination" = "install_ramble.yml" - } - - bucket_md5 = substr(md5("${var.project_id}.${var.deployment_name}"), 0, 8) - # Max bucket name length is 63, so truncate deployment_name if necessary. - # The string "-ramble-scripts-" is 16 characters and bucket_md5 is 8 characters, - # leaving 63-16-8=39 chars for deployment_name. - bucket_name = "${substr(var.deployment_name, 0, 39)}-ramble-scripts-${local.bucket_md5}" - runners = [local.install_ramble_deps_runner, local.install_ramble_runner, local.python_reqs_runner] - - combined_runner = { - "type" = "shell" - "content" = module.startup_script.startup_script - "destination" = "ramble-install-and-setup.sh" - } - -} - -resource "google_storage_bucket" "bucket" { - project = var.project_id - name = local.bucket_name - uniform_bucket_level_access = true - location = var.region - storage_class = "REGIONAL" - labels = local.labels -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.runners - gcs_bucket_path = "gs://${google_storage_bucket.bucket.name}" -} - -resource "local_file" "debug_file_shell_install" { - content = local.script_content - filename = "${path.module}/debug_install.yml" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf deleted file mode 100644 index e587470eac..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/outputs.tf +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "Ramble installation script." - value = module.startup_script.startup_script -} - -output "controller_startup_script" { - description = "Ramble installation script, duplicate for SLURM controller." - value = module.startup_script.startup_script -} - -output "ramble_runner" { - description = <<-EOT - Runner to be used with startup-script module or passed to ramble-execute module. - - installs Ramble dependencies - - installs Ramble - - generates profile.d script to enable access to Ramble - This is safe to run in parallel by multiple machines. - EOT - value = local.combined_runner -} - -output "ramble_path" { - description = "Location ramble is installed into." - value = var.install_dir -} - -output "ramble_ref" { - description = "Git ref the ramble install is checked out to use" - value = var.ramble_ref -} - -output "gcs_bucket_path" { - description = "Bucket containing the startup scripts for Ramble, to be reused by ramble-execute module." - value = "gs://${google_storage_bucket.bucket.name}" -} - -output "ramble_profile_script_path" { - description = "Path to Ramble profile script." - value = var.ramble_profile_script_path -} - -output "system_user_name" { - description = "The system user used to install Ramble. It can be reused by ramble-execute module to execute Ramble commands." - value = var.system_user_name -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml deleted file mode 100644 index b7905bbe9e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/scripts/install_ramble_deps.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Create python virtual env for a tool - become: yes - hosts: localhost - vars: - virtualenv_path: ${virtualenv_path} - tasks: - - name: Install dependencies through system package manager - ansible.builtin.package: - name: - - python3 - - python3-pip - - git - register: package - changed_when: package.changed - retries: 5 - delay: 10 - until: package is success - - - name: Create virtualenv for tool - # Python 3.6 is minimum we wish to support due to ease of installation on - # CentOS 7 and Rocky Linux 8. pip 21.3.1 is the *maximum* version of pip - # supported by 3.6. Additionally, recent versions of pip are necessary for - # proper dependency resolution of real-world problems with google-cloud-* - # (and third-party) Python packages (20.3+ probably effective minimum). - ansible.builtin.pip: - name: pip>=21.3.1 - virtualenv: "{{ virtualenv_path }}" - virtualenv_command: /usr/bin/python3 -m venv - - - name: Add google-cloud-storage to virtualenv - ansible.builtin.pip: - name: google-cloud-storage - virtualenv: "{{ virtualenv_path }}" - virtualenv_command: /usr/bin/python3 -m venv diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl deleted file mode 100644 index ea14780a58..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/install_ramble_python_deps.yml.tftpl +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Install Python Requirements - hosts: localhost - vars: - install_dir: ${install_dir} - virtualenv_path: ${virtualenv_path} - tasks: - - - name: Install dependencies - ansible.builtin.pip: - requirements: "{{ install_dir }}/requirements.txt" - virtualenv: "{{ virtualenv_path }}" - virtualenv_command: /usr/bin/python3 -m venv diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl deleted file mode 100644 index ca48a5afa0..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/templates/ramble_setup.yml.tftpl +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Install Software - hosts: localhost - vars: - sw_name: ${sw_name} - profile_script: ${profile_script} - install_dir: ${install_dir} - git_url: ${git_url} - git_ref: ${git_ref} - chmod_mode: ${chmod_mode} - system_user_name: ${system_user_name} - system_user_uid: ${system_user_uid} - system_user_gid: ${system_user_gid} - finalize_setup_script: ${finalize_setup_script} - profile_script_path: ${profile_script_path} - tasks: - - name: Print software name - ansible.builtin.debug: - msg: "Running installation for software: {{ sw_name }}" - - - name: Add profile script for software - ansible.builtin.copy: - dest: "{{ profile_script_path }}" - mode: '0644' - content: "{{ profile_script }}" - when: profile_script - - - name: Look up user to use for install - block: - - - name: Check if user already exists - ansible.builtin.getent: - database: passwd - key: "{{ system_user_name }}" - - - name: Look up existing user details - ansible.builtin.user: - name: "{{ system_user_name }}" - register: system_user - - rescue: - - name: User did not exist, create group for system user - ansible.builtin.group: - name: "{{ system_user_name }}" - gid: "{{ system_user_gid }}" - system: true - register: system_group - - - name: Create system user - ansible.builtin.user: - name: "{{ system_user_name }}" - comment: "{{ sw_name }} installation" - uid: "{{ system_user_uid }}" - group: "{{ system_group.name }}" - system: true - register: system_user - - - name: Create parent of install directory - ansible.builtin.file: - path: "{{ install_dir | dirname }}" - state: directory - - - name: Set lock dir - ansible.builtin.set_fact: - lock_dir: "{{ install_dir | dirname }}/.install_{{ sw_name }}_lock" - - - name: Acquire lock - ansible.builtin.command: - mkdir "{{ lock_dir }}" - register: lock_out - changed_when: lock_out.rc == 0 - failed_when: false - - - name: Add hostname to lock_dir - ansible.builtin.file: - path: "{{ lock_dir }}/{{ ansible_hostname }}" - state: touch - when: lock_out.rc == 0 - - - name: Clone branch or tag into installation directory - ansible.builtin.command: git clone --branch {{ git_ref }} {{ git_url }} {{ install_dir }} - failed_when: false - register: clone_res - when: lock_out.rc == 0 - - - name: Clone commit hash into installation directory - ansible.builtin.command: "{{ item }}" - with_items: - - git clone {{ git_url }} {{ install_dir }} - - git -C {{ install_dir }} checkout {{ git_ref }} - when: lock_out.rc == 0 and clone_res.rc != 0 - - - name: Transfer ownership to system user - ansible.builtin.file: - path: "{{ install_dir }}" - owner: "{{ system_user.name }}" - group: "{{ system_user.group }}" - recurse: true - follow: false - when: lock_out.rc == 0 - - - name: Finalize setup - ansible.builtin.shell: "{{ finalize_setup_script }}" - when: lock_out.rc == 0 and finalize_setup_script - become: true - become_user: "{{ system_user.name }}" - - - name: Apply chmod - ansible.builtin.file: - path: "{{ install_dir }}" - mode: "{{ chmod_mode | default(omit, true) }}" - recurse: true - follow: false - when: (lock_out.rc == 0) and (chmod_mode != None) - - - name: Release lock - ansible.builtin.file: - path: "{{ lock_dir }}/done" - state: touch - when: lock_out.rc == 0 - - - name: Wait for lock - block: - - name: Wait for lock - ansible.builtin.wait_for: - path: "{{ lock_dir }}/done" - state: present - timeout: 600 - sleep: 10 - when: lock_out.rc != 0 - - rescue: - - name: Timed out on waiting for lock, get lock directory contents - ansible.builtin.find: - paths: "{{ lock_dir }}" - register: lock_dir_contents - - - name: Print lock directory contents, it should contain name of host that is holding lock - ansible.builtin.debug: - msg: "{{ lock_dir_contents.files|map(attribute='path')|map('basename')|list }}" - - - name: Failed to get lock - ansible.builtin.fail: - msg: "Timeout waiting on lock for ${sw_name}, exiting" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/variables.tf deleted file mode 100644 index 0d3a8eed05..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/variables.tf +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created." - type = string -} - -variable "install_dir" { - description = "Destination directory of installation of Ramble." - default = "/apps/ramble" - type = string -} - -variable "ramble_url" { - description = "URL for Ramble repository to clone." - default = "https://github.com/GoogleCloudPlatform/ramble" - type = string -} - -variable "ramble_ref" { - description = "Git ref to checkout for Ramble." - default = "develop" - type = string -} - -variable "chmod_mode" { - description = <<-EOT - Mode to chmod the Ramble clone to. Defaults to `""` (i.e. do not modify). - For usage information see: - https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode - EOT - default = "" - type = string - nullable = false -} - -variable "system_user_name" { - description = "Name of system user that will perform installation of Ramble. It will be created if it does not exist." - default = "ramble" - type = string - nullable = false -} - -variable "system_user_uid" { - description = "UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary." - default = 1104762904 - type = number - nullable = false -} - -variable "system_user_gid" { - description = "GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762904 is arbitrary." - default = 1104762904 - type = number - nullable = false -} - -variable "ramble_virtualenv_path" { - description = "Virtual environment path in which to install Ramble Python interpreter and other dependencies" - default = "/usr/local/ramble-python" - type = string -} - -variable "deployment_name" { - description = "Name of deployment, used to name bucket containing startup script." - type = string -} - -variable "region" { - description = "Region to place bucket containing startup script." - type = string -} - -variable "labels" { - description = "Key-value pairs of labels to be added to created resources." - type = map(string) -} - -variable "ramble_profile_script_path" { - description = "Path to the Ramble profile.d script. Created by this module" - type = string - default = "/etc/profile.d/ramble.sh" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/versions.tf deleted file mode 100644 index 936b4a5b80..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/ramble-setup/versions.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/README.md deleted file mode 100644 index 8cbb75fb42..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/README.md +++ /dev/null @@ -1,141 +0,0 @@ -## Description - -This module creates a script that defines a software build using Spack and -performs any additional customization to a Spack installation. - -There are two main variable inputs that can be used to define a Spack build: -`data_files` and `commands`. - -- `data_files`: Any files specified will be transferred to the machine running - outputted script. Data file `content` can be defined inline in the blueprint - or can point to a `source`, an absolute local path of a file. This can be used - to transfer environment definition files, config definition files, GPG keys, - or software licenses. `data_files` are transferred before `commands` are run. -- `commands`: A script that is run. This can be used to perform actions such as - installation of compilers & packages, environment creation, adding a build - cache, and modifying the spack configuration. - -## Example - -The `spack-execute` module should `use` a `spack-setup` module. This will -prepend the installation of Spack and its dependencies to the build. Then -`spack-execute` can be used by a module that takes `startup-script` as an input. - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: spack-build - source: community/modules/scripts/spack-execute - use: [spack-setup] - settings: - commands: | - spack install gcc@10.3.0 target=x86_64 - - - id: builder-vm - source: modules/compute/vm-instance - use: [network1, spack-build] -``` - -To see a full example of this module in use, see the [hpc-slurm-gromacs.yaml] example. - -[hpc-slurm-gromacs.yaml]: ../../../examples/hpc-slurm-gromacs.yaml - -### Using with `startup-script` module - -The `spack-runner` output can be used by the `startup-script` module. - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: spack-build - source: community/modules/scripts/spack-execute - use: [spack-setup] - settings: - commands: | - spack install gcc@10.3.0 target=x86_64 - - - id: startup-script - source: modules/scripts/startup-script - settings: - runners: - - $(spack-build.spack-runner) - - type: shell - destination: "my-script.sh" - content: echo 'hello world' - - - id: workstation - source: modules/compute/vm-instance - use: [network1, startup-script] -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [local](#requirement\_local) | >= 2.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [local](#provider\_local) | >= 2.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [local_file.debug_file_ansible_execute](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [commands](#input\_commands) | String of commands to run within this module | `string` | `null` | no | -| [data\_files](#input\_data\_files) | A list of files to be transferred prior to running commands.
It must specify one of 'source' (absolute local file path) or 'content' (string).
It must specify a 'destination' with absolute path where file should be placed. | `list(map(string))` | `[]` | no | -| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing spack scripts. | `string` | n/a | yes | -| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | n/a | yes | -| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | -| [log\_file](#input\_log\_file) | Defines the logfile that script output will be written to | `string` | `"/var/log/spack.log"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [region](#input\_region) | Region to place bucket containing spack scripts. | `string` | n/a | yes | -| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script. Created by an instance of spack-setup.
Can be defined explicitly, or by chaining an instance of a spack-setup module
through a `use` setting. | `string` | n/a | yes | -| [spack\_runner](#input\_spack\_runner) | Runner from previous spack-setup or spack-execute to be chained with scripts generated by this module. | 
object({
    type        = string
    content     = string
    destination = string
  })
| n/a | yes | -| [system\_user\_name](#input\_system\_user\_name) | Name of the system user used to execute commands. Generally passed from the spack-setup module. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [controller\_startup\_script](#output\_controller\_startup\_script) | Spack startup script, duplicate for SLURM controller. | -| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for spack, to be reused by spack-execute module. | -| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to the Spack profile.d script. | -| [spack\_runner](#output\_spack\_runner) | Single runner that combines scripts from this module and any previously chained spack-execute or spack-setup modules. | -| [startup\_script](#output\_startup\_script) | Spack startup script. | -| [system\_user\_name](#output\_system\_user\_name) | The system user used to execute commands. | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/main.tf deleted file mode 100644 index 04ebcf7d49..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/main.tf +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "spack-execute", ghpc_role = "scripts" }) -} - -locals { - commands_content = var.commands == null ? "echo 'no spack commands provided'" : indent(4, yamlencode(var.commands)) - - execute_contents = templatefile( - "${path.module}/templates/execute_commands.yml.tpl", - { - pre_script = ". ${var.spack_profile_script_path}" - log_file = var.log_file - commands = local.commands_content - system_user_name = var.system_user_name - } - ) - - data_runners = [for data_file in var.data_files : merge(data_file, { type = "data" })] - - execute_md5 = substr(md5(local.execute_contents), 0, 4) - execute_runner = { - type = "ansible-local" - content = local.execute_contents - destination = "spack_execute_${local.execute_md5}.yml" - } - - runners = concat([var.spack_runner], local.data_runners, [local.execute_runner]) - - # Destinations should be unique while also being known at time of apply - combined_unique_string = join("\n", [for runner in local.runners : runner["destination"]]) - combined_md5 = substr(md5(local.combined_unique_string), 0, 4) - combined_runner = { - type = "shell" - content = module.startup_script.startup_script - destination = "combined_install_spack_${local.combined_md5}.sh" - } -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.runners - gcs_bucket_path = var.gcs_bucket_path -} - -resource "local_file" "debug_file_ansible_execute" { - content = local.execute_contents - filename = "${path.module}/debug_execute_${local.execute_md5}.yml" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/outputs.tf deleted file mode 100644 index 4a52532d51..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/outputs.tf +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "Spack startup script." - value = module.startup_script.startup_script -} - -output "controller_startup_script" { - description = "Spack startup script, duplicate for SLURM controller." - value = module.startup_script.startup_script -} - -output "spack_runner" { - description = "Single runner that combines scripts from this module and any previously chained spack-execute or spack-setup modules." - value = local.combined_runner -} - -output "gcs_bucket_path" { - description = "Bucket containing the startup scripts for spack, to be reused by spack-execute module." - value = var.gcs_bucket_path -} - -output "spack_profile_script_path" { - description = "Path to the Spack profile.d script." - value = var.spack_profile_script_path -} - -output "system_user_name" { - description = "The system user used to execute commands." - value = var.system_user_name -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl deleted file mode 100644 index 0e98f3aa2c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/templates/execute_commands.yml.tpl +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Execute Commands - hosts: localhost - vars: - pre_script: ${pre_script} - log_file: ${log_file} - commands: ${commands} - system_user_name: ${system_user_name} - tasks: - - name: Execute command block - block: - - name: Print commands to be executed - ansible.builtin.debug: - msg: "{{ commands.split('\n') | ansible.builtin.to_nice_yaml }}" - - - name: Streaming log info - ansible.builtin.debug: - msg: | - Logs from commands will not be printed here until success (or failure) - Streaming logs can be found at {{ log_file }} - - - name: Ensure user can write to log file - ansible.builtin.file: - path: "{{ log_file }}" - state: touch - owner: "{{ system_user_name }}" - - - name: Execute commands - ansible.builtin.shell: | - set -eo pipefail - { - {{ pre_script }} - echo " === Starting commands ===" - {{ commands }} - echo " === Finished commands ===" - } 2>&1 | tee -a {{ log_file }} - args: - executable: /bin/bash - register: output - become: true - become_user: "{{ system_user_name }}" - - always: - - name: Print commands output - ansible.builtin.debug: - var: output.stdout_lines diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/variables.tf deleted file mode 100644 index 851cd1aed8..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/variables.tf +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created." - type = string -} - -variable "deployment_name" { - description = "Name of deployment, used to name bucket containing spack scripts." - type = string -} - -variable "region" { - description = "Region to place bucket containing spack scripts." - type = string -} - -variable "labels" { - description = "Key-value pairs of labels to be added to created resources." - type = map(string) -} - -variable "log_file" { - description = "Defines the logfile that script output will be written to" - default = "/var/log/spack.log" - type = string -} - -variable "data_files" { - description = <<-EOT - A list of files to be transferred prior to running commands. - It must specify one of 'source' (absolute local file path) or 'content' (string). - It must specify a 'destination' with absolute path where file should be placed. - EOT - type = list(map(string)) - default = [] - validation { - condition = alltrue([for r in var.data_files : substr(r["destination"], 0, 1) == "/"]) - error_message = "All destinations must be absolute paths and start with '/'." - } - validation { - condition = alltrue([ - for r in var.data_files : - can(r["content"]) != can(r["source"]) - ]) - error_message = "A data_file must specify either 'content' or 'source', but never both." - } - validation { - condition = alltrue([ - for r in var.data_files : - lookup(r, "content", lookup(r, "source", null)) != null - ]) - error_message = "A data_file must specify a non-null 'content' or 'source'." - } -} - -variable "commands" { - description = "String of commands to run within this module" - type = string - default = null -} - -variable "spack_runner" { - description = "Runner from previous spack-setup or spack-execute to be chained with scripts generated by this module." - type = object({ - type = string - content = string - destination = string - }) -} - -variable "system_user_name" { - description = "Name of the system user used to execute commands. Generally passed from the spack-setup module." - type = string -} - -variable "gcs_bucket_path" { - description = "The GCS path for storage bucket and the object, starting with `gs://`." - type = string -} - -variable "spack_profile_script_path" { - description = <<-EOT - Path to the Spack profile.d script. Created by an instance of spack-setup. - Can be defined explicitly, or by chaining an instance of a spack-setup module - through a `use` setting. - EOT - type = string -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/versions.tf deleted file mode 100644 index 09583c3d43..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-execute/versions.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 1.0.0" - required_providers { - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/README.md deleted file mode 100644 index 01d3e6d389..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/README.md +++ /dev/null @@ -1,382 +0,0 @@ -## Description - -This module can be used to setup and install Spack on a VM. To actually run -Spack commands to install other software use the -[spack-execute](../spack-execute/) module. - -This module generates a script that performs the following: - -1. Install system dependencies needed for Spack -1. Clone Spack into a predefined directory -1. Check out a specific version of Spack - -There are several options on how to consume the outputs of this module: - -> [!IMPORTANT] -> Breaking changes between after v1.21.0. `spack-install` module replaced by -> `spack-setup` and `spack-execute` modules. -> [Details Below](#deprecations-and-breaking-changes) - -## Examples - -### `use` `spack-setup` with `spack-execute` - -This will prepend the `spack-setup` script to the `spack-execute` commands. - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: spack-build - source: community/modules/scripts/spack-execute - use: [spack-setup] - settings: - commands: | - spack install gcc@10.3.0 target=x86_64 - - - id: builder - source: modules/compute/vm-instance - use: [network1, spack-build] -``` - -### `use` `spack-setup` with `vm-instance` or Slurm module - -This will run `spack-setup` scripts on the downstream compute resource. - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: spack-installer - source: modules/compute/vm-instance - use: [network1, spack-setup] -``` - -OR - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - use: [network1, partition1, spack-setup] -``` - -### Build `starup-script` with `spack-runner` output - -This will use the generated `spack-setup` script as one step in `startup-script`. - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - - - id: startup-script - source: modules/scripts/startup-script - settings: - runners: - - $(spack-setup.spack-runner) - - type: shell - destination: "my-script.sh" - content: echo 'hello world' - - - id: workstation - source: modules/compute/vm-instance - use: [network1, startup-script] -``` - -To see a full example of this module in use, see the [hpc-slurm-gromacs.yaml] example. - -[hpc-slurm-gromacs.yaml]: ../../../examples/hpc-slurm-gromacs.yaml - -## Environment Setup - -### Activating Spack - -[Spack installation] produces a setup script that adds `spack` to your `PATH` as -well as some other command-line integration tools. This script can be found at -`/share/spack/setup-env.sh`. This script will be automatically -added to bash startup by any machine that runs the `spack_runner`. - -If you have multiple machines that all want to use the same shared Spack -installation you can just have both machines run the `spack_runner`. - -[Spack installation]: https://spack-tutorial.readthedocs.io/en/latest/tutorial_basics.html#installing-spack - -### Managing Spack Python dependencies - -Spack is configured with [SPACK_PYTHON] to ensure that Spack itself uses a -Python virtual environment with a supported copy of Python with the package -`google-cloud-storage` pre-installed. This enables Spack to use mirrors and -[build caches][builds] on Google Cloud Storage. It does not configure Python -packages *inside* Spack virtual environments. If you need to add more Python -dependencies for Spack itself, use the `spack python` command: - -```shell -sudo -i spack python -m pip install package-name -``` - -[SPACK_PYTHON]: https://spack.readthedocs.io/en/latest/getting_started.html#shell-support -[builds]: https://spack.readthedocs.io/en/latest/binary_caches.html - -## Spack Permissions - -### System `spack` user is created - Default - -By default this module will create a `spack` linux user and group with -consistent UID and GID. This user and group will own the Spack installation. To -allow a user to manually add Spack packages to the system Spack installation, -you can add the user to the spack group: - -```sh -sudo usermod -a -G spack -``` - -Log out and back in so the group change will take effect, then `` will -be able to call `spack install `. - -> [!NOTE] -> A background persistent SSH connections may prevent the group change from -> taking effect. - -You can use the `system_user_name`, `system_user_uid`, and `system_user_gid` to -customize the name and ids of the system user. While unlikely, it is possible -that the default `system_user_uid` or `system_user_gid` could conflict with -existing UIDs. - -### Use and existing user - -Alternatively, if `system_user_name` is a user already on the system, then this -existing user will be used for Spack installation. - -#### OS Login User - -If OS Login is enabled (default for most Cluster Toolkit modules) then you can -provide an OS Login user name: - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - settings: - system_user_name: username_company_com -``` - -This will work even if the user has not yet logged onto the machine. When the -specified user does log on to the machine they will be able to call -`spack install` without any further configuration. - -#### Pre-configured user - -You can also use a startup script to configure a user: - -```yaml - - id: spack-setup - source: community/modules/scripts/spack-setup - settings: - system_user_name: special-user - - - id: startup - source: modules/scripts/startup-script - settings: - runners: - - type: shell - destination: "create_user.sh" - content: | - #!/bin/bash - sudo useradd -u 799 special-user - sudo groupadd -g 922 org-group - sudo usermod -g org-group special-user - - $(spack-setup.spack_runner) - - - id: spack-vms - source: modules/compute/vm-instance - use: [network1, startup] - settings: - name_prefix: spack-vm - machine_type: n2d-standard-2 - instance_count: 5 -``` - -### Chaining spack installations - -If there is a need to have a non-root user to install spack packages it is -recommended to create a separate installation for that user and chain Spack installations -([Spack docs](https://spack.readthedocs.io/en/latest/chain.html#chaining-spack-installations)). - -Steps to chain Spack installations: - -1. Get the version of the system Spack: - - ```sh - $ spack --version - - 0.20.0 (e493ab31c6f81a9e415a4b0e0e2263374c61e758) - # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - # Note commit hash and use in next step - ``` - -1. Clone a new spack installation: - - ```sh - git clone -c feature.manyFiles=true https://github.com/spack/spack.git /spack - git -C /spack checkout - ``` - -1. Point the new Spack installation to the system Spack installation. Create a - file at `/spack/etc/spack/upstreams.yaml` with the following - contents: - - ```yaml - upstreams: - spack-instance-1: - install_tree: /sw/spack/opt/spack/ - ``` - -1. Add the following line to your `.bashrc` to make sure the new `spack` is in - your `PATH`. - - ```sh - . /spack/share/spack/setup-env.sh - ``` - -## Deprecations and Breaking Changes - -The old `spack-install` module has been replaced by the `spack-setup` and -`spack-execute` modules. Generally this change strives to allow for a more -flexible definition of a Spack build by using native Spack commands. - -For every deprecated variable from `spack-install` there is documentation on how -to perform the equivalent action using `commands` and `data_files`. The -documentation can be found on the [inputs table](#inputs) below. - -Below is a simple example of the same functionality shown before and after the -breaking changes. - -```yaml - # Before - - id: spack-install - source: community/modules/scripts/spack-install - settings: - install_dir: /sw/spack - compilers: - - gcc@10.3.0 target=x86_64 - packages: - - intel-mpi@2018.4.274%gcc@10.3.0 - -- id: spack-startup - source: modules/scripts/startup-script - settings: - runners: - - $(spack.install_spack_deps_runner) - - $(spack.install_spack_runner) -``` - -```yaml - # After - - id: spack-setup - source: community/modules/scripts/spack-setup - settings: - install_dir: /sw/spack - - - id: spack-execute - source: community/modules/scripts/spack-execute - use: [spack-setup] - settings: - commands: | - spack install gcc@10.3.0 target=x86_64 - spack load gcc@10.3.0 target=x86_64 - spack compiler find --scope site - spack install intel-mpi@2018.4.274%gcc@10.3.0 - -- id: spack-startup - source: modules/scripts/startup-script - settings: - runners: - - $(spack-execute.spack-runner) -``` - -Although the old `spack-install` module will no longer be maintained, it is -still possible to use the old module in a blueprint by referencing an old -version from GitHub. Note the source line in the following example. - -```yaml - - id: spack-install - source: github.com/GoogleCloudPlatform/hpc-toolkit//community/modules/scripts/spack-install?ref=v1.22.1&depth=1 -``` - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [google](#requirement\_google) | >= 4.42 | -| [local](#requirement\_local) | >= 2.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [local](#provider\_local) | >= 2.0.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | -| [local_file.debug_file_shell_install](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [chmod\_mode](#input\_chmod\_mode) | `chmod` to apply to the Spack installation. Adds group write by default. Set to `""` (empty string) to prevent modification.
For usage information see:
https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode | `string` | `"g+w"` | no | -| [configure\_for\_google](#input\_configure\_for\_google) | When true, the spack installation will be configured to pull from Google's Spack binary cache. | `bool` | `true` | no | -| [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing startup script. | `string` | n/a | yes | -| [install\_dir](#input\_install\_dir) | Directory to install spack into. | `string` | `"/sw/spack"` | no | -| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | -| [spack\_profile\_script\_path](#input\_spack\_profile\_script\_path) | Path to the Spack profile.d script. Created by this module | `string` | `"/etc/profile.d/spack.sh"` | no | -| [spack\_ref](#input\_spack\_ref) | Git ref to checkout for spack. | `string` | `"v0.20.0"` | no | -| [spack\_url](#input\_spack\_url) | URL to clone the spack repo from. | `string` | `"https://github.com/spack/spack"` | no | -| [spack\_virtualenv\_path](#input\_spack\_virtualenv\_path) | Virtual environment path in which to install Spack Python interpreter and other dependencies | `string` | `"/usr/local/spack-python"` | no | -| [system\_user\_gid](#input\_system\_user\_gid) | GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary. | `number` | `1104762903` | no | -| [system\_user\_name](#input\_system\_user\_name) | Name of system user that will perform installation of Spack. It will be created if it does not exist. | `string` | `"spack"` | no | -| [system\_user\_uid](#input\_system\_user\_uid) | UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary. | `number` | `1104762903` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [controller\_startup\_script](#output\_controller\_startup\_script) | Spack installation script, duplicate for SLURM controller. | -| [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for spack, to be reused by spack-execute module. | -| [spack\_path](#output\_spack\_path) | Path to the root of the spack installation | -| [spack\_profile\_script\_path](#output\_spack\_profile\_script\_path) | Path to the Spack profile.d script. | -| [spack\_runner](#output\_spack\_runner) | Runner to be used with startup-script module or passed to spack-execute module.
- installs Spack dependencies
- installs Spack
- generates profile.d script to enable access to Spack
This is safe to run in parallel by multiple machines. Use in place of deprecated `setup_spack_runner`. | -| [startup\_script](#output\_startup\_script) | Spack installation script. | -| [system\_user\_name](#output\_system\_user\_name) | The system user used to install Spack. It can be reused by spack-execute module to install spack packages. | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/main.tf deleted file mode 100644 index d45f5d1be3..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/main.tf +++ /dev/null @@ -1,120 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "spack-setup", ghpc_role = "scripts" }) -} - -locals { - profile_script = <<-EOF - SPACK_PYTHON=${var.spack_virtualenv_path}/bin/python3 - if [ -f ${var.install_dir}/share/spack/setup-env.sh ]; then - test -t 1 && echo "Running Spack setup, this may take a moment on first login." - . ${var.install_dir}/share/spack/setup-env.sh - fi - EOF - - supported_cache_versions = ["v0.19.0", "v0.20.0"] - cache_version = contains(local.supported_cache_versions, var.spack_ref) ? var.spack_ref : "latest" - add_google_mirror_script = !var.configure_for_google ? "" : <<-EOF - if ! spack mirror list | grep -q google_binary_cache; then - spack mirror add --scope site google_binary_cache gs://spack/${local.cache_version} - spack buildcache keys --install --trust - fi - EOF - - finalize_setup_script = <<-EOF - set -e - . ${var.spack_profile_script_path} - spack config --scope site add 'packages:all:permissions:read:world' - spack config --scope site add 'packages:all:permissions:write:group' - spack gpg init - spack compiler find --scope site - ${local.add_google_mirror_script} - # perform fast install to make sure Spack is fully initialized - spack install xz - spack uninstall --yes-to-all xz - EOF - - script_content = templatefile( - "${path.module}/templates/spack_setup.yml.tftpl", - { - sw_name = "spack" - profile_script = indent(4, yamlencode(local.profile_script)) - install_dir = var.install_dir - git_url = var.spack_url - git_ref = var.spack_ref - chmod_mode = var.chmod_mode - system_user_name = var.system_user_name - system_user_uid = var.system_user_uid - system_user_gid = var.system_user_gid - finalize_setup_script = indent(4, yamlencode(local.finalize_setup_script)) - profile_script_path = var.spack_profile_script_path - } - ) - - install_spack_deps_runner = { - "type" = "ansible-local" - "source" = "${path.module}/scripts/install_spack_deps.yml" - "destination" = "install_spack_deps.yml" - "args" = "-e virtualenv_path=${var.spack_virtualenv_path}" - } - install_spack_runner = { - "type" = "ansible-local" - "content" = local.script_content - "destination" = "install_spack.yml" - } - - bucket_md5 = substr(md5("${var.project_id}.${var.deployment_name}.${local.script_content}"), 0, 8) - # Max bucket name length is 63, so truncate deployment_name if necessary. - # The string "-spack-scripts-" is 15 characters and bucket_md5 is 8 characters, - # leaving 63-15-8=40 chars for deployment_name. Using 39 so it has the same prefix as the - # ramble-setup module's GCS bucket. - bucket_name = "${substr(var.deployment_name, 0, 39)}-spack-scripts-${local.bucket_md5}" - runners = [local.install_spack_deps_runner, local.install_spack_runner] - - combined_runner = { - "type" = "shell" - "content" = module.startup_script.startup_script - "destination" = "spack-install-and-setup.sh" - } -} - -resource "google_storage_bucket" "bucket" { - project = var.project_id - name = local.bucket_name - uniform_bucket_level_access = true - location = var.region - storage_class = "REGIONAL" - labels = local.labels -} - -module "startup_script" { - source = "../../../../modules/scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.runners - gcs_bucket_path = "gs://${google_storage_bucket.bucket.name}" -} - -resource "local_file" "debug_file_shell_install" { - content = local.script_content - filename = "${path.module}/debug_install.yml" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml deleted file mode 100644 index 2ada34471f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - storage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/outputs.tf deleted file mode 100644 index d94b9757db..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/outputs.tf +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "Spack installation script." - value = module.startup_script.startup_script -} - -output "controller_startup_script" { - description = "Spack installation script, duplicate for SLURM controller." - value = module.startup_script.startup_script -} - -output "spack_path" { - description = "Path to the root of the spack installation" - value = var.install_dir -} - -output "spack_runner" { - description = <<-EOT - Runner to be used with startup-script module or passed to spack-execute module. - - installs Spack dependencies - - installs Spack - - generates profile.d script to enable access to Spack - This is safe to run in parallel by multiple machines. Use in place of deprecated `setup_spack_runner`. - EOT - value = local.combined_runner -} - -output "gcs_bucket_path" { - description = "Bucket containing the startup scripts for spack, to be reused by spack-execute module." - value = "gs://${google_storage_bucket.bucket.name}" -} - -output "spack_profile_script_path" { - description = "Path to the Spack profile.d script." - value = var.spack_profile_script_path -} - -output "system_user_name" { - description = "The system user used to install Spack. It can be reused by spack-execute module to install spack packages." - value = var.system_user_name -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml deleted file mode 100644 index b7905bbe9e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/scripts/install_spack_deps.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Create python virtual env for a tool - become: yes - hosts: localhost - vars: - virtualenv_path: ${virtualenv_path} - tasks: - - name: Install dependencies through system package manager - ansible.builtin.package: - name: - - python3 - - python3-pip - - git - register: package - changed_when: package.changed - retries: 5 - delay: 10 - until: package is success - - - name: Create virtualenv for tool - # Python 3.6 is minimum we wish to support due to ease of installation on - # CentOS 7 and Rocky Linux 8. pip 21.3.1 is the *maximum* version of pip - # supported by 3.6. Additionally, recent versions of pip are necessary for - # proper dependency resolution of real-world problems with google-cloud-* - # (and third-party) Python packages (20.3+ probably effective minimum). - ansible.builtin.pip: - name: pip>=21.3.1 - virtualenv: "{{ virtualenv_path }}" - virtualenv_command: /usr/bin/python3 -m venv - - - name: Add google-cloud-storage to virtualenv - ansible.builtin.pip: - name: google-cloud-storage - virtualenv: "{{ virtualenv_path }}" - virtualenv_command: /usr/bin/python3 -m venv diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl deleted file mode 100644 index ca48a5afa0..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/templates/spack_setup.yml.tftpl +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Install Software - hosts: localhost - vars: - sw_name: ${sw_name} - profile_script: ${profile_script} - install_dir: ${install_dir} - git_url: ${git_url} - git_ref: ${git_ref} - chmod_mode: ${chmod_mode} - system_user_name: ${system_user_name} - system_user_uid: ${system_user_uid} - system_user_gid: ${system_user_gid} - finalize_setup_script: ${finalize_setup_script} - profile_script_path: ${profile_script_path} - tasks: - - name: Print software name - ansible.builtin.debug: - msg: "Running installation for software: {{ sw_name }}" - - - name: Add profile script for software - ansible.builtin.copy: - dest: "{{ profile_script_path }}" - mode: '0644' - content: "{{ profile_script }}" - when: profile_script - - - name: Look up user to use for install - block: - - - name: Check if user already exists - ansible.builtin.getent: - database: passwd - key: "{{ system_user_name }}" - - - name: Look up existing user details - ansible.builtin.user: - name: "{{ system_user_name }}" - register: system_user - - rescue: - - name: User did not exist, create group for system user - ansible.builtin.group: - name: "{{ system_user_name }}" - gid: "{{ system_user_gid }}" - system: true - register: system_group - - - name: Create system user - ansible.builtin.user: - name: "{{ system_user_name }}" - comment: "{{ sw_name }} installation" - uid: "{{ system_user_uid }}" - group: "{{ system_group.name }}" - system: true - register: system_user - - - name: Create parent of install directory - ansible.builtin.file: - path: "{{ install_dir | dirname }}" - state: directory - - - name: Set lock dir - ansible.builtin.set_fact: - lock_dir: "{{ install_dir | dirname }}/.install_{{ sw_name }}_lock" - - - name: Acquire lock - ansible.builtin.command: - mkdir "{{ lock_dir }}" - register: lock_out - changed_when: lock_out.rc == 0 - failed_when: false - - - name: Add hostname to lock_dir - ansible.builtin.file: - path: "{{ lock_dir }}/{{ ansible_hostname }}" - state: touch - when: lock_out.rc == 0 - - - name: Clone branch or tag into installation directory - ansible.builtin.command: git clone --branch {{ git_ref }} {{ git_url }} {{ install_dir }} - failed_when: false - register: clone_res - when: lock_out.rc == 0 - - - name: Clone commit hash into installation directory - ansible.builtin.command: "{{ item }}" - with_items: - - git clone {{ git_url }} {{ install_dir }} - - git -C {{ install_dir }} checkout {{ git_ref }} - when: lock_out.rc == 0 and clone_res.rc != 0 - - - name: Transfer ownership to system user - ansible.builtin.file: - path: "{{ install_dir }}" - owner: "{{ system_user.name }}" - group: "{{ system_user.group }}" - recurse: true - follow: false - when: lock_out.rc == 0 - - - name: Finalize setup - ansible.builtin.shell: "{{ finalize_setup_script }}" - when: lock_out.rc == 0 and finalize_setup_script - become: true - become_user: "{{ system_user.name }}" - - - name: Apply chmod - ansible.builtin.file: - path: "{{ install_dir }}" - mode: "{{ chmod_mode | default(omit, true) }}" - recurse: true - follow: false - when: (lock_out.rc == 0) and (chmod_mode != None) - - - name: Release lock - ansible.builtin.file: - path: "{{ lock_dir }}/done" - state: touch - when: lock_out.rc == 0 - - - name: Wait for lock - block: - - name: Wait for lock - ansible.builtin.wait_for: - path: "{{ lock_dir }}/done" - state: present - timeout: 600 - sleep: 10 - when: lock_out.rc != 0 - - rescue: - - name: Timed out on waiting for lock, get lock directory contents - ansible.builtin.find: - paths: "{{ lock_dir }}" - register: lock_dir_contents - - - name: Print lock directory contents, it should contain name of host that is holding lock - ansible.builtin.debug: - msg: "{{ lock_dir_contents.files|map(attribute='path')|map('basename')|list }}" - - - name: Failed to get lock - ansible.builtin.fail: - msg: "Timeout waiting on lock for ${sw_name}, exiting" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/variables.tf deleted file mode 100644 index 85baeec401..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/variables.tf +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created." - type = string -} - -# spack-setup variables - -variable "install_dir" { - description = "Directory to install spack into." - type = string - default = "/sw/spack" -} - -variable "spack_url" { - description = "URL to clone the spack repo from." - type = string - default = "https://github.com/spack/spack" -} - -variable "spack_ref" { - description = "Git ref to checkout for spack." - type = string - default = "v0.20.0" -} - -variable "configure_for_google" { - description = "When true, the spack installation will be configured to pull from Google's Spack binary cache." - type = bool - default = true -} - - -variable "chmod_mode" { - description = <<-EOT - `chmod` to apply to the Spack installation. Adds group write by default. Set to `""` (empty string) to prevent modification. - For usage information see: - https://docs.ansible.com/ansible/latest/collections/ansible/builtin/file_module.html#parameter-mode - EOT - default = "g+w" - type = string - nullable = false -} - -variable "system_user_name" { - description = "Name of system user that will perform installation of Spack. It will be created if it does not exist." - default = "spack" - type = string - nullable = false -} - -variable "system_user_uid" { - description = "UID used when creating system user. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary." - default = 1104762903 - type = number - nullable = false -} - -variable "system_user_gid" { - description = "GID used when creating system user group. Ignored if `system_user_name` already exists on system. Default of 1104762903 is arbitrary." - default = 1104762903 - type = number - nullable = false -} - -variable "spack_virtualenv_path" { - description = "Virtual environment path in which to install Spack Python interpreter and other dependencies" - default = "/usr/local/spack-python" - type = string -} - -variable "deployment_name" { - description = "Name of deployment, used to name bucket containing startup script." - type = string -} - -variable "region" { - description = "Region to place bucket containing startup script." - type = string -} - -variable "labels" { - description = "Key-value pairs of labels to be added to created resources." - type = map(string) -} - -variable "spack_profile_script_path" { - description = "Path to the Spack profile.d script. Created by this module" - type = string - default = "/etc/profile.d/spack.sh" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/versions.tf deleted file mode 100644 index ff1180fc1b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/spack-setup/versions.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.0.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/README.md deleted file mode 100644 index 1a3e4cf7e6..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/README.md +++ /dev/null @@ -1,187 +0,0 @@ -## Description - -Creates a containerised Guacamole instance. Works with the Rocky, Debian and Ubuntu images shown in blueprint example below. - -### Secret Manager Integration - -The VDI module supports flexible Secret Manager integration: - -- **Default Behavior**: Secrets are stored in the deployment project -- **Automatic Password Generation**: If no `secret_name` is provided, random passwords are generated and stored -- **Existing Secret Retrieval**: Provide `secret_name` to use existing secrets from Secret Manager -- **Cross-Project Secrets**: Use `secret_project` to specify a different GCP project if providing secrets - -## Basic Example (Guacamole) - -```yaml -blueprint_name: vdi-test - -vars: - deployment_name: vdi-test - project_id: * project name here * - region: us-central1 - zone: us-central1-a - -deployment_groups: -- group: primary - modules: - - id: network1 - source: modules/network/vpc - settings: - extra_iap_ports: [8080] - firewall_rules: - - name: allow-guacamole-8080-ext - description: Allow external ingress to Guacamole on TCP port 8080 - direction: INGRESS - ranges: ["0.0.0.0/0"] - allow: - - protocol: tcp - ports: ["8080"] - - - id: enable-apis - source: community/modules/project/service-enablement - settings: - gcp_service_list: - - secretmanager.googleapis.com - - storage.googleapis.com - - compute.googleapis.com - - - id: vdi-setup - source: community/modules/scripts/vdi-setup - settings: - vnc_flavor: tigervnc - vdi_tool: guacamole - user_provision: local_users - vdi_user_group: vdiusers - vdi_resolution: 1920x1080 - vdi_users: - # Alice: password generated and saved to Secret Manager in deployment project - - username: alice - port: 5901 - # Bob: existing password is retrieved from Secret Manager in deployment project - - username: bob - port: 5902 - secret_name: a-password-for-bob - # Charlie: password retrieved from Secret Manager in a different project - - username: charlie - port: 5903 - secret_name: charlie-password - secret_project: another-project-id - - - id: guac_vm - source: modules/compute/vm-instance - settings: - instance_image: - # Several supported image families: - family: hpc-rocky-linux-8 - project: cloud-hpc-image-public - #family: debian-11 - #project: debian-cloud - #family: ubuntu-2204-lts - #project: ubuntu-os-cloud - machine_type: e2-highcpu-8 - tags: ["guacamole"] - use: - - network1 - - vdi-setup -``` - -[NVIDIA vWS drivers](https://cloud.google.com/compute/docs/gpus/grid-drivers-table) will be automatically installed on [supported machine types](https://cloud.google.com/compute/docs/gpus#gpu-virtual-workstations). - -Important note: Before deploying the above example you would need to ensure the `secret_name` exists in your local project, and that your service account has sufficient access permissions if retrieving the secret from a separate `secret_project`. The following two example commands show how you can create the two example user's respective secrets: - -```bash -# Create secrets for each user in the deployment project -echo -n "BobPassword123" | gcloud secrets create a-password-for-bob --data-file=- - -# Or create secrets in a different project (if using secret_project) -echo -n "CharliePassword123" | gcloud secrets create charlie-password --data-file=- --project=another-project-id -``` - -### Accessing Guacamole VDI - -After deployment, you can access the VDI in several ways: - -1. **Guacamole Web Interface**: - - Access web interface: - - http://$VM_PUBLIC_IP:8080/guacamole/#/ - - Note: It is not advisable to serve Guacamole directly from a public IP in production environments. You should consider placing the VDI behind a reverse proxy, load balancer, or tunnel to it directly over IAP (see below). - - - Admin credentials: - - Username: `guacadmin` - - Password: Retrieve the `webapp-server...` secret from Secret Manager - -2. **User VDI Access**: - - Each user's credentials are stored in Secret Manager - - Secret names follow the pattern: `vdi-user-password-{username}-{deployment_name}` for auto-generated passwords - - Or use the custom secret name specified in the `vdi_users` configuration - -3. **IAP Tunnel** (for development/testing): - - ```bash - gcloud compute start-iap-tunnel vdi-test-0 8080 \ - --local-host-port=localhost:8080 \ - --zone=us-central1-a - ``` - - - Guacamole will then be accessible from http://localhost:8080/guacamole/ - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [archive](#requirement\_archive) | ~> 2.0 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [archive](#provider\_archive) | ~> 2.0 | -| [google](#provider\_google) | >= 3.83 | -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | -| [terraform_data.input_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [archive_file.roles_tar](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | n/a | yes | -| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | -| [user\_provision](#input\_user\_provision) | User type to create (local\_users supported. os-login to do. | `string` | `"local_users"` | no | -| [vdi\_resolution](#input\_vdi\_resolution) | Desktop resolution for VNC sessions (e.g. 1920x1080). | `string` | `"1920x1080"` | no | -| [vdi\_tool](#input\_vdi\_tool) | VDI tool to deploy (guacamole currently supported). | `string` | `"guacamole"` | no | -| [vdi\_user\_group](#input\_vdi\_user\_group) | Unix group to create/use for VDI users. | `string` | `"vdiusers"` | no | -| [vdi\_users](#input\_vdi\_users) | List of VDI users to configure. Passwords are handled securely by the Ansible roles: if secret\_name is provided, the password is fetched from Secret Manager; if neither password nor secret\_name is provided, a random password is generated and stored in Secret Manager. If secret\_project is provided, it specifies the GCP project where the secret is stored (defaults to the deployment project). | 
list(object({
    username       = string
    port           = number
    secret_name    = optional(string)
    secret_project = optional(string)
  }))
| `[]` | no | -| [vdi\_webapp\_port](#input\_vdi\_webapp\_port) | Port to serve the Webapp interface from if applicable | `string` | `"8080"` | no | -| [vnc\_flavor](#input\_vnc\_flavor) | The VNC server flavor to use (tigervnc currently supported) | `string` | `"tigervnc"` | no | -| [vnc\_port\_max](#input\_vnc\_port\_max) | Maximum valid VNC port. | `number` | `5999` | no | -| [vnc\_port\_min](#input\_vnc\_port\_min) | Minimum valid VNC port. | `number` | `5901` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [guacamole\_admin\_password\_secret](#output\_guacamole\_admin\_password\_secret) | The name of the Secret Manager secret containing the Guacamole admin password | -| [guacamole\_admin\_username](#output\_guacamole\_admin\_username) | The admin username for Guacamole | -| [startup\_script](#output\_startup\_script) | Combined startup script that installs VDI (VNC, Guacamole, users). | -| [vdi\_runner](#output\_vdi\_runner) | Shell runner wrapping Ansible playbook + roles (for custom-image or direct use). | -| [vdi\_user\_credentials](#output\_vdi\_user\_credentials) | Map of VDI user credentials stored in Secret Manager | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/main.tf deleted file mode 100644 index 38aec83f2c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/main.tf +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "vdi-setup", ghpc_role = "scripts" }) -} - -# Create a tar.gz of roles/ directory -data "archive_file" "roles_tar" { - type = "tar.gz" - source_dir = "${path.module}/roles" - output_path = "${path.module}/roles.tar.gz" -} - -# Generate vars content using templatefile -locals { - vdi_vars_content = templatefile("${path.module}/templates/vars.yaml.tftpl", { - deployment_name = var.deployment_name - project_id = var.project_id - user_provision = var.user_provision - vnc_flavor = var.vnc_flavor - vdi_tool = var.vdi_tool - vdi_user_group = var.vdi_user_group - vdi_webapp_port = var.vdi_webapp_port - vdi_resolution = var.vdi_resolution - vdi_users = var.vdi_users - }) -} - -# Assemble runners -locals { - runners = [ - # Install dependencies - { - type = "shell" - destination = "install-deps.sh" - content = <<-EOT - #!/bin/bash - set -eux - /usr/local/ghpc-venv/bin/python3 -m pip install requests google-auth docker - ansible-galaxy collection install google.cloud - EOT - }, - # Stage roles.tar.gz - { - type = "data" - source = data.archive_file.roles_tar.output_path - destination = "/tmp/vdi/roles.tar.gz" - }, - - # Unpack into /tmp/vdi/roles - { - type = "shell" - destination = "unpack_roles.sh" - content = <<-EOT - #!/bin/bash - set -eux - mkdir -p /tmp/vdi/roles - tar xzf /tmp/vdi/roles.tar.gz -C /tmp/vdi/roles - EOT - }, - - # write out vars file as YAML - { - type = "data" - content = local.vdi_vars_content - destination = "/tmp/vdi/vars.yaml" - }, - - # Run the rendered playbook via ansible-local - { - type = "ansible-local" - content = templatefile("${path.module}/templates/install.yaml.tftpl", - { - roles = ["base_os", "secret_manager", "user_provision", "vnc", "vdi_tool"], - } - ) - destination = "/tmp/vdi/install.yaml" - # Debug mode: - # args = "--extra-vars @/tmp/vdi/vars.yaml -v --extra-vars debug=true" - args = "--extra-vars @/tmp/vdi/vars.yaml" - }, - # Todo: another runner here to delete /tmp/vdi afterwards? - ] - - bucket_name = "${substr(var.deployment_name, 0, 39)}-vdi-scripts-${substr(md5(var.deployment_name), 0, 8)}" -} - -# Bucket to stage runners -resource "google_storage_bucket" "bucket" { - labels = local.labels - project = var.project_id - name = local.bucket_name - location = var.region - uniform_bucket_level_access = true - storage_class = "REGIONAL" -} - -# Use the startup-script module to push and execute them -module "startup_script" { - labels = local.labels - source = "../../../../modules/scripts/startup-script" - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - - runners = local.runners - gcs_bucket_path = "gs://${google_storage_bucket.bucket.name}" - - docker = { - enabled = true - } -} - -# Expose the combined startup script -locals { - combined_runner = { - type = "shell" - content = module.startup_script.startup_script - destination = "install-vdi-and-setup.sh" - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/outputs.tf deleted file mode 100644 index 4e8fa9f21e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/outputs.tf +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "startup_script" { - description = "Combined startup script that installs VDI (VNC, Guacamole, users)." - value = module.startup_script.startup_script -} - -output "vdi_runner" { - description = "Shell runner wrapping Ansible playbook + roles (for custom-image or direct use)." - value = local.combined_runner -} - -output "guacamole_admin_username" { - description = "The admin username for Guacamole" - value = "guacadmin" -} - -output "guacamole_admin_password_secret" { - description = "The name of the Secret Manager secret containing the Guacamole admin password" - value = "webapp-server-password-${var.deployment_name}" -} - -output "vdi_user_credentials" { - description = "Map of VDI user credentials stored in Secret Manager" - value = { - for user in var.vdi_users : user.username => { - username = user.username - port = user.port - secret_name = user.secret_name != null ? user.secret_name : "vdi-user-password-${user.username}-${var.deployment_name}" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles.tar.gz b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles.tar.gz deleted file mode 100644 index 2d0c250eb491e56b53f76f02d13545662f5a9bfa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8732 zcmV+%BIDg3iwFP!00000|LlEhbK5ww=zi9(=#?ug`^MDElAY0(_pV|q8P~};IhLK- zQ%On%ldyy_MQ{O7wyxt-_b1%ncz?+$Jops#vK3`ALwraq5|2is(cNftH?I-L1IlXx z@mpSV@HIhRb&AIBUp80j^?LoV)l&c0>-Fsa&BOL#__u!EXw?r}e}Vc2VVGM2E)e@m z{n=w>C-;Mt9XO%>l#$^`K(pR#!&^#+E{1pSPD(qa9e77<>~ZWs;5nFq7-8u8$R6SF z%|3j@j1%fXvs#B;S)puRmG}N$+JPwzV2q~VQ2_zRAV!444RJBxdmH-#hzB+u`z}GA zjp2@nks8KyQZ4Pkf0-`mwLruJ1UB`jKyQ-u00{-KDkCBMPOWx#cUMIU+A3wknyYK_ z+Pkxp-g&=gHLG>i=E`$1=fLp41H!Na*Hb{g?-CnbyBJ(_2b2LCGK?J%R04d*h#=l@ zA2_`ccZlKA4mgAhMy>;q!d3_#;i(GL1LQ%u+lRA$8D4k$XZ`)s4*Ys{`Sbg$OZc^W zanU`$JnQx0{RN!7KR-RYJbQoMhxc!wd;VYWA7|&M`+$iUVFvi#XA*wOfJi)IMF2%-Vl7u#)Qk&TXz^URO@gNh#h{Pf<0U(c)YZq6q1L6weRXG-e8)S=4Nv_#2AOj!Sw`hoY z2R>EqZ*1JE>_g?AV>ZFex~FVt*_2`V(76t{b>C=IzNp3%&mQ=*>G@4zatS%aNGvQaxMJ zCC6*G@f>mkrj10)z`>Ig^88zgD({92XE5mCiG1dg>-$C{`3v8R#LuBS@tp9tTrtPO zoMTTAQ$>@4?z#N% zkRl3r37^b5e}R{?^M=RAvTDu+61XEFk?@uP7#563#5C$900+Bh+JS@FIox9AVHZXi zIhb*^V_4v+oncyFNwgDp@^9mqfu9g;tbpcDT!px#4AyM)s zYSZx;c}_f%J$B~74VLBHjyYHsC$5$wrVYbLNNG|FIICR~FKo-le=G9;LUIy#Cl5q(lz@zk{1~{m$b++!Q;@2G z6qTty^5tdf6c(SvCaN<_ca#$4sbiNc>7$p{70mIgP+7FfvcftPddrVdmdO9$I`BmB zZO{Mh`e8dO|J$vjLjG@~6!O22|67s&SBjJTna=;I8Ggi!+)Ux^$q7u%e!0dsl;IFf zM2lO6>N%6iR@lo|Wv&LR5veaj!Ublk-oVrv*duJ;ni}>9c|+_B?nc<_ zzzuRaPQ!YNuY=+9Ky_pnF?z;)*n#qDJ$B&XVHWl(7Y+@CTGv(8l#t}UmxhvE5dCUk zCOcr52X5Ha4I=KMH93(;0CE$~GY4_N1KY;f!OlGT5lF<-^^q8@1veiPLq|E(V!%ql zlbYZv6cYWCy_eHnDn3U9sC_3%Bo~`iW~a8k>L{L??1BrPY#+@^p{J@V%=U4XLbRgA z1yCf2bzv`e~t@6@T*EVI;9=8B)n6r*)9#7>QGUcl)W{%_JP1PhuVq(@tvU` zSQElTfL!a2vRiHdSgV)XkP*kI@Z1t|CG}}F^7?s;1-w1^0Apm2h=)N;eGE5@j^%+7 zIY=OgJg?@SKh@dOkXzU+go}6rGY`2TaKD)MwMnC<6cf+JH8Z*;M^`;&PpigzC=~Ny zC)OL~g4Qt`wxksn5ioCHQ%_(|bYQC7?vf`$e_+NNsPQ6P^zH>iwg5h&Or$Wf+;1<_ zsd3!OJgpj_s@!*p*satmd!MYv7p>f8p$NynL1W@h&6=zOQ#i8jU=BnE605TJsh$BO zI&GzBgK*%1?^7l>aN5|h6e7Y0Mx~nE@j=v$1}zP)vK-+Kx9kfLnfW?Gn2t`Alu{_QO@RA!{eEZ=l#px2>gu0nFKg`yf zMJdrO21*I*z#=Pr$tdXN_z?ON$_3%oB%sPGRfAA5mFKKSX$JDd{mv%9kDUnCgI{1KoFCT%T!XL-^6#zssglS*Pgt)srBmFWpK{3`jxU(*hNh!3L`>L;l>~G5_e)R0Of9FEAbhCY6 zbV3xPmldC=Q>9mIc9T_^Ef=f)`%sSl1SPHiZN%MorY!USZ69Uhe-Dn1nuY$ijZ)}; zh5on6_}^Tu;^{%a|NPgDjVt?=e@Z@O-G75<-%W-1ZY{1i5!OrD(7JClOh=Z3C)iEB z2n2DhF|rxu*|1)misHTU9VDch<5NxqWmB-#CjVQgi9qi|?_J9C7pUh>R}6&3K6WjJ zU5v~Xd5-6Gpg~i~i5U)>3@Uh+uHl5(3w3G6?cCf&< zvS1<4$#`YP_s_;$Bbg`?H!A{FYeMDzw_+Xzl3g#exoeHfx7+I9WLb|AqP*6tm$kPBIEg( zbOUI7k?dn`RlRR3jYcKyEug(O4U&_~hxMfLF`aPUOgcYXP`*HSIlU?G&H1p!4xAR0 z4K}e8%d~7yd8kTHOHz&vwRVbKtgN#hhgDf{b5%u!rQxNs?<$8$n{bwR72J=aXikqhq6~na*QOSLv?svJ=tteyI7VdO0y7X{X zste0<2=`qyO)!ph;AgV>Q=>mqL+Y^Cb?`T*izvQx7Q_r1HHa%q+~0vU^~Wn5+YF1r z7!f(%Zn_t%Of2`%tMl zcvABM*98}QLoqr&Xv$L$PJ?=fuaWNqa>m449bDaN zpjv%%kWX}YDg2rTDvS^xNvh>&g603F*4)t61rw27=}F|yPoCuwAMIB72sPV>l|A?h zf$w8Bd#`|OTT-l`w^(0g^)==EqU8{DKb$R%x-CB^>{d>n*w{k0O#@Hx@Z?idAeA>A zi7QxuQ&$`s0RuH!mI(U3Et@;1f|DQR>8kW%YEF=Nz;Q<`%yL391XLC87$O0Fl^TYdoEBgkbQgS+DOytNzm$fF1ci90PJ0M$XhEA4-vd z*+&N7I-68X2VKVr*F4%|p#?5BiMxAu&-xo}%&mRKsp zA2725u41T`XvBWHrqa6sdyLp4aEZz_mJI^rDw;~IoO%=nRk@V<8vcnx2C`b{G9}+A zWrR_a`At2UAl3mQ@VJ!n;nSmIC+=ntUZsTYd`tDlnWh=Qq8>GX+SEV$mZ%#>;{9I2 z<^OOc*Zz}FVNV`ong91;Jsbajbkr>Tzqe8f`%hv2d9M8@YlkpL=}@~+3In+Tk9rt< zMkhq4s7(uuMCA^YCk|0kF{Ok^NmZ3kjS4bf6;0-9QLQ35PPih(P^cuTq@YdLO1QQO zi6Jkcsc{-GTVq#=@foB+iK9@V{TkBZZhRlLmsE2z?F@jS(=)HR zNMJ5HS`l5zDzs%acd4O?kK;tsIr%F>)hk1S@D&6jV>mvBvc^q0ts{OsJ=5zOuQ;RQ z3^}Z#B`RGbZ6S?7*61;MdR1lQQ+l|A6MA?){0YF7MjiL54|B3YvtFNX_kAiQuQI`v zOu&u9T==X!V+BdrKjvW_b|Z+xe)8fq zz7tQ0ttQVDVn5)K`kUEP)}o}ope)ujVqzy=R$TocQtBCm^oKUnqOyARn{vQ#FXt#2O(30Om&+NR@i;1Myc z>h*A|TA?3s!H74k?1%dGt9@+>LGH0S3hnLZjb;4o2VO9~#;npYGKVs`uZSnWg7!~| zPd|m_aUUL1*aC=ueKCJMK7MzrwqD;bwVSd3y3|IlrmIc30WR_XJ7^tbzyEi5*lZQ{ z-))q_{#)38H)a2gGem~gAr8BCzy%#k*k)BGz86QhDwentYTxhQV|^rYz-u>}mAw?yCc#$R5%r8W0_*_Cf`fkt zoZ!~uFg5G(Ut+b|dmNfAx$?%Ji^?D^8HW%3sLk#(duqD%vbrqXUeL<_# z#~$aS!7ZN3=4+y&Mw|iTsQfJ_ekdodA0=$e8ss!Wx{d?-a}5d|g36z{o29AyJ-TGo(OHDL13MKR*vTR3l_Vy@kLE$6(Eqkk3jMFp z|2C!n84aMK6rhS0Ea`mwTGo_eJxf(B<*?q6BWF90^gG>qWWviziJa?@|L_0P7iJ-clvpA+z_E*MsTR%ZzN($pewXJ*&z`jNz4$WDa!bAWLMv68xkt8` z<722K2CjrNe&2_8bV$O)_=~3*RbFwXln+$E!gi+_j#~LM$nvL@T=}0`tLKlgME*A# z^_lN~HX6nEKeti}`CrKYXUhNYO~OTaMv=l-Q0j&sfDVw2#?-}|%mKc{|GU*_X6^s2 zqoa1g|F=;J{$KF_4WIu$P%efH_y6;qx>;A>kRT$m%M#4>u`TD!5Zix^!P8jvvPZSF zjn|kt8KwH%Xe^r1@5pG|ZdbyShcQ*ws4rrsXF1n33G&>Raw5)k_B+9+)V@^#Tq>kI z&XhKL#jh&p50rIg@RsR@@Gv*^{H~G*Tt_ueVo7o~VVvBtCu!N4{U)?-HV9!ez(=tC ziimcGGSE2)k9Az)H3sEIlLOryB4ThuS#l|B6U1sP@M?O%DyJqTogiaO1JQwlgg5Rb z<{})@rk;mwK_*1x4QA8v7aq8`62_WzUaZ{C}&ygk8O7R4L zs4ez@?*;aFn2_7hM;?&AF?M^52ijXA}# z;1_g@!68G;#qQodjK;{8|6^fS0ZkBbO?00uCl|f$WeCjKEvre8Oz1sS7Z?x$(8N`)_5$I56%iww!$>=yGhy`7vzFOz#M zCohPD9bzl)X5jX)xTEZL1;9;Bv!m3o0}qSzw~lq4Ff~OcPAt6CPgRnN(O-|D9=&b9n$kdT-? zfaOEGbC!bmbo>2Z-(Q>_=cbD=UJS_wr>}E5 z%wyNe9a*W%F1fD7EH!a7S;XZvE-dT`HuWj-M7Rx&zm&zWIv4{r=Q9(EEN z>F4+T%k%Cpz2o$Cf zj&cgb=u#m$k>zOlVi^rSI+E+7K5;RJK-;m^*BE4{#^|?#p>?jQGht3d*Tyr)e7s*J z$dDt&AxAoprnnLOaZ2X?Z^NFSx-9emXt%T9e>gZgD*QjUQi}V(;{I<__kZQ+`);8i zDbHpvOnVrlQvXRll=9w->Q-_FIj6rMdqljvK|DIv?^YNjs+%{+uaw<#pNgC@?#!zd zh6toCp#oArYt|Mzx^Wx>bu6F7<1JdDau2INbw zRMNKf0rmtDQ|R_Vr+w=8-t|r{K|dZ4ZIOR-@&1?iq;J}&H^2VeyXdJ?HXK7GJx&F> z=cgd1zIu1n?_E?%04gt&hwl=OWAW%`=l$NrC7hjKzE6yosJWY2l_Q&*>B z`q+JU)$7A<7LKdkj*6u9v;G*gMl&e4~Rc$ zRgvQ;iSzhaiPC26R(#5YX=Sc?l5a!oni@%DC;LdG#C>>3xz?$gs^XqRvPSprocZo$ z7bhXv)>xsa73@Y7zCVNVs8(`e5*Jct161QK%**szRMY*2w^o>qc{LuR(}_W)*D9Z1 z{Bj_7PrTNN!S}NZZ@QOQk2b+YuX|eA+uyBReK_r2_SB!Jy>~tHXZQ4%v-7k5W$&W0 zr`JHjwws>+Uda$<EJh+mQaR_NC$O3IVEkRM8VT~ZU^mU? z+H5AG{ff`X1PN^2;%Oz`61_^CrMoQ8D&_q!`A!-BJMQ$qp`6ORlTP>t@HO;D+%13p zEc^KU`Sa&RwJPO(fO4g>uqP&aT3Ex=V|nF0Id4;O6TEX>vfC>El)!D3hboHR1|R)) z`d^OzpBsAdoH3T@|MkPe?EUXiqgB}dw^A%CbTf{-8|qidZ%85q*!uLL5 zSOSn>QkkL7uim{2+p0LLigAHTDzf`f>Hpld)Ms*aWslzNnm?5rP2$f|Bu({D8)OI* z&;FLKAF3>~|F#;f%>Hj49TfK8t&~z>04@x`8+QP?Bty(To}c8nO@w*BGj0>J8N-i{ z6G_7J#~6vZW2ijitB#?QlK&|U*)ji5EzCy8SZ4oiw6pm?4h|2R1^?ehDcFC({-4YK z6Q4V_sdqz$0aGm!ABNOV6Y=RE&rg73A(6;u8MtKC>KLl~+f2^Du#E)xtxK?m5f46@)}PF({S*(0^XD%>y(33<3JvpHG9RuO(@ z2{r{n_^H`NR8}?z{8aLsoCG`HmT=}yFrt~o6YDG_`t|3RPY}x|^u3jxqRf<({vRKj zHaf;K`_HWZ-@#$C$p5#MQt1DM{$Ke2{dr5w|MSjVn;ByX|35ft9A>}&(Ksx=|F)G< z@c)AUZ#e&}Jej2IK#9+R&HY}-Oy<|$x}CfVemE6lwX4y^&5$@IDNSCyMD`r2IJaV((U>-W#zpAUZN zo_F8&E>iDKUtjeHVFmej_w@9l*AE@F@IAKmdGP+m#;tFHWu2b)2}kLirHSs^6{lIS zKFcyKVtYi>Q_Ron2w!31+3JY{)(!k)rDfFh%lXL<*X!UK+jOj5AQr7jgsV5?9t^k& zM~S_^nf;l7ZR50ZnA(w2gr!Z(n*Xev=Tz!D8GGdN>FIXgW_npOrmW8}r*2@rqHX$! zpFPpQqV&IH=!ZtdMs_+Qb;k_J5<9_5W(O>V^N;R?5Q*r5x#R{(8|hBD{Dkb&Ol4PKd3)a}SS?$Km6o z67}!`u;<90UKEVtM=Epn|LncuGsal9|7ZO_j~dORV*hWWe7f?8_)+-Ag=qW(C( zpVrr{5r23=O5GcQ*)gATmkz4}u^|>=if8k)RQlA{r1Pb8IrZ`J4Q02)8|^`5(Jqxf8Lz=FrC*UJu=9F)JPuqz)QKU4{Cls(_b5yEf8?$4^f8w0|90{$ zp!fel%l`xb0RR7tShsZm G6afIrxTI- - {%- if vnc_flavor|lower == 'tigervnc' -%} - tigervnc-standalone-server - {%- elif vnc_flavor|lower == 'tightvnc' -%} - tightvnc-standalone-server - {%- else -%} - tightvnc-standalone-server - {%- endif %} - xorg_dev_packages: ['pkg-config', 'xserver-xorg-dev', 'libx11-dev', 'libxext-dev', 'libglvnd-dev', 'build-essential'] - -- name: Install desktop prerequisites & VNC package (Debian) - ansible.builtin.package: - name: "{{ gui_packages + [vnc_pkg] + xorg_dev_packages }}" - state: present - retries: 10 - delay: 3 - -- name: Install kernel headers for running kernel (Debian) - ansible.builtin.package: - name: "linux-headers-{{ ansible_kernel }}" - state: present - retries: 10 - delay: 3 diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml deleted file mode 100644 index f148682825..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# NVIDIA vWS (GRID) driver setup. See: -# https://cloud.google.com/compute/docs/gpus#gpu-virtual-workstations -# https://cloud.google.com/compute/docs/gpus/grid-drivers-table - ---- - -- name: Get GCP machine type from metadata server - ansible.builtin.uri: - url: http://metadata.google.internal/computeMetadata/v1/instance/machine-type - method: GET - headers: - Metadata-Flavor: Google - return_content: yes - register: gcp_machine_type - -- name: Extract short machine type - ansible.builtin.set_fact: - gcp_machine_type: "{{ gcp_machine_type.content.split('/')[-1] }}" - -- name: Extract machine family from machine type - ansible.builtin.set_fact: - machine_family: "{{ gcp_machine_type.split('-')[0] }}" - -- name: Check if machine type is supported for NVIDIA vWS - ansible.builtin.set_fact: - is_nvidia_vws_supported: "{{ machine_family in ['g2', 'n1'] }}" - -# Additional drivers for other machine type families can go here -- name: Set driver download URL based on machine type - ansible.builtin.set_fact: - driver_url: >- - {%- if is_nvidia_vws_supported -%} - https://storage.googleapis.com/nvidia-drivers-us-public/GRID/vGPU18.1/NVIDIA-Linux-x86_64-570.133.20-grid.run - {%- else -%} - - {%- endif %} - -# Include distribution-specific GPU driver setup -- name: Include Ubuntu GPU driver setup - ansible.builtin.include_tasks: gpu_driver_setup/ubuntu.yaml - when: - - ansible_facts.distribution == 'Ubuntu' - - is_nvidia_vws_supported - -- name: Include Rocky Linux GPU driver setup - ansible.builtin.include_tasks: gpu_driver_setup/rocky.yaml - when: - - ansible_facts.distribution == 'Rocky' - - is_nvidia_vws_supported - -- name: Include Debian GPU driver setup - ansible.builtin.include_tasks: gpu_driver_setup/debian.yaml - when: - - ansible_facts.distribution == 'Debian' - - is_nvidia_vws_supported diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/debian.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/debian.yaml deleted file mode 100644 index 7bd4f3699c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/debian.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -# Block for NVIDIA vWS driver setup (Debian) -- block: - - name: Download NVIDIA vWS driver - ansible.builtin.uri: - url: "{{ driver_url }}" - method: GET - dest: /tmp/nvidia-driver.run - mode: '0755' - status_code: - - 200 - - 304 - - name: Install NVIDIA vWS driver (Debian) - ansible.builtin.command: - cmd: /tmp/nvidia-driver.run --dkms --silent - creates: /usr/bin/nvidia-smi diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/rocky.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/rocky.yaml deleted file mode 100644 index 260ca90d19..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/rocky.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -# Block for NVIDIA vWS driver setup (Rocky Linux) -- block: - - name: Install NVIDIA Vulkan ICD loader (Rocky Linux) - ansible.builtin.package: - name: vulkan-loader - state: present - retries: 10 - delay: 3 - - name: Download NVIDIA vWS driver - ansible.builtin.uri: - url: "{{ driver_url }}" - method: GET - dest: /tmp/nvidia-driver.run - mode: '0755' - status_code: - - 200 - - 304 - - name: Install NVIDIA vWS driver (Rocky Linux) - ansible.builtin.command: - cmd: /tmp/nvidia-driver.run --dkms --silent - creates: /usr/bin/nvidia-smi diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/ubuntu.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/ubuntu.yaml deleted file mode 100644 index 8cef6585a0..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/gpu_driver_setup/ubuntu.yaml +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -# Block for NVIDIA vWS driver setup (Ubuntu) -- block: - - name: Download NVIDIA vWS driver - ansible.builtin.uri: - url: "{{ driver_url }}" - method: GET - dest: /tmp/nvidia-driver.run - mode: '0755' - status_code: - - 200 - - 304 - - - name: Verify GCC version before driver installation - ansible.builtin.shell: gcc --version | head -1 - register: gcc_version_check - changed_when: false - - - name: Debug GCC version before driver installation - ansible.builtin.debug: - msg: "GCC version before driver installation: {{ gcc_version_check.stdout }}" - - - name: Install NVIDIA vWS driver (Ubuntu) - ansible.builtin.command: - cmd: /tmp/nvidia-driver.run --dkms --silent - creates: /usr/bin/nvidia-smi - register: nvidia_install_result - failed_when: false - - - name: Check if NVIDIA driver installation succeeded - ansible.builtin.stat: - path: /usr/bin/nvidia-smi - register: nvidia_smi_check - - - name: Debug NVIDIA installation result - ansible.builtin.debug: - msg: - - "NVIDIA installer exit code: {{ nvidia_install_result.rc }}" - - "nvidia-smi exists: {{ nvidia_smi_check.stat.exists }}" - - "Installation stdout: {{ nvidia_install_result.stdout }}" - - "Installation stderr: {{ nvidia_install_result.stderr }}" - - - name: Fail if NVIDIA driver installation failed - ansible.builtin.fail: - msg: "NVIDIA driver installation failed. Check /var/log/nvidia-installer.log for details." - when: - - nvidia_install_result.rc != 0 - - not nvidia_smi_check.stat.exists - - - name: Verify NVIDIA driver installation - ansible.builtin.command: nvidia-smi - register: nvidia_smi_test - changed_when: false - when: nvidia_smi_check.stat.exists - - - name: Debug NVIDIA driver verification - ansible.builtin.debug: - msg: "NVIDIA driver verification: {{ nvidia_smi_test.stdout }}" - when: nvidia_smi_check.stat.exists diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml deleted file mode 100644 index 209155873b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Gather distribution facts - ansible.builtin.setup: - gather_subset: platform - -# GPU / vWS driver setup -- name: Check if NVIDIA GPU is present - ansible.builtin.shell: lspci | grep -i nvidia - register: nvidia_gpu_present - failed_when: false - -# Include distribution-specific tasks -- name: Include Ubuntu-specific tasks - ansible.builtin.include_tasks: ubuntu.yaml - when: ansible_facts.distribution == 'Ubuntu' - -- name: Include Rocky Linux specific tasks - ansible.builtin.include_tasks: rocky.yaml - when: ansible_facts.distribution == 'Rocky' - -- name: Include Debian-specific tasks - ansible.builtin.include_tasks: debian.yaml - when: ansible_facts.distribution == 'Debian' - -# Common tasks for all distributions -- name: Ensure requests package is installed - pip: - name: requests - state: present - -- name: Ensure Python & tooling are installed - ansible.builtin.package: - name: - - python3 - - python3-pip - - jq - state: present - -# GPU / vWS driver setup -- name: Provision GPU setup - ansible.builtin.include_tasks: gpu_driver_setup.yaml - when: nvidia_gpu_present.rc == 0 diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/rocky.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/rocky.yaml deleted file mode 100644 index 132e53fa89..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/rocky.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Set Rocky Linux specific variables - ansible.builtin.set_fact: - gui_packages: ['@Xfce','@base-x'] - vnc_pkg: >- - {%- if vnc_flavor|lower == 'tigervnc' -%} - tigervnc-server - {%- elif vnc_flavor|lower == 'tightvnc' -%} - tightvnc-server - {%- else -%} - tightvnc-standalone-server - {%- endif %} - xorg_dev_packages: ['pkgconfig', 'xorg-x11-server-devel', 'xorg-x11-util-macros', 'libglvnd-devel'] - -- name: Update repository cache & install EPEL (Rocky Linux only) - ansible.builtin.package: - name: epel-release - state: present - update_cache: yes - retries: 10 - delay: 3 - -- name: Install desktop prerequisites & VNC package (Rocky Linux) - ansible.builtin.package: - name: "{{ gui_packages + [vnc_pkg] + xorg_dev_packages }}" - state: present - retries: 10 - delay: 3 diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml deleted file mode 100644 index e467a0cca1..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Set Ubuntu specific variables - ansible.builtin.set_fact: - gui_packages: ['xfce4', 'xserver-xorg-core'] - vnc_pkg: >- - {%- if vnc_flavor|lower == 'tigervnc' -%} - tigervnc-standalone-server - {%- elif vnc_flavor|lower == 'tightvnc' -%} - tightvnc-standalone-server - {%- else -%} - tightvnc-standalone-server - {%- endif %} - xorg_dev_packages: ['pkg-config', 'xserver-xorg-dev', 'libx11-dev', 'libxext-dev', 'libglvnd-dev'] - -- name: Install desktop prerequisites & VNC package (Ubuntu) - ansible.builtin.package: - name: "{{ gui_packages + [vnc_pkg] + xorg_dev_packages }}" - state: present - retries: 10 - delay: 3 - -- name: Install kernel headers for running kernel (Ubuntu) - ansible.builtin.package: - name: "linux-headers-{{ ansible_kernel }}" - state: present - retries: 10 - delay: 3 - -# GCC version detection and installation for NVIDIA driver compatibility -- name: Get kernel build GCC version - ansible.builtin.shell: cat /proc/version | grep -o 'gcc-[0-9]*' | head -1 - register: kernel_gcc_version - changed_when: false - -- name: Extract GCC major version from kernel - ansible.builtin.set_fact: - required_gcc_version: "{{ kernel_gcc_version.stdout.split('-')[1] if kernel_gcc_version.stdout else '11' }}" - -- name: Debug kernel GCC version extraction - ansible.builtin.debug: - msg: - - "Raw kernel GCC version: {{ kernel_gcc_version.stdout }}" - - "Extracted required GCC version: {{ required_gcc_version }}" - -- name: Get current GCC version - ansible.builtin.shell: gcc --version | head -1 | grep -o '[0-9]*\.[0-9]*' | head -1 - register: current_gcc_version - changed_when: false - -- name: Extract current GCC major version - ansible.builtin.set_fact: - current_gcc_major: "{{ current_gcc_version.stdout.split('.')[0] if current_gcc_version.stdout else '11' }}" - -- name: Debug current GCC version extraction - ansible.builtin.debug: - msg: - - "Raw current GCC version: {{ current_gcc_version.stdout }}" - - "Extracted current GCC major: {{ current_gcc_major }}" - -- name: Debug GCC version information - ansible.builtin.debug: - msg: - - "Kernel was built with GCC {{ required_gcc_version }}" - - "Current system GCC version: {{ current_gcc_major }}" - - "GCC update needed: {{ required_gcc_version != current_gcc_major }}" - -- name: Install required GCC version if different from current - block: - - name: Install GCC {{ required_gcc_version }} - ansible.builtin.package: - name: "gcc-{{ required_gcc_version }}" - state: present - retries: 10 - delay: 3 - - - name: Install G++ {{ required_gcc_version }} - ansible.builtin.package: - name: "g++-{{ required_gcc_version }}" - state: present - retries: 10 - delay: 3 - - - name: Set GCC {{ required_gcc_version }} as default - ansible.builtin.alternatives: - name: gcc - path: "/usr/bin/gcc-{{ required_gcc_version }}" - priority: "{{ required_gcc_version }}" - link: /usr/bin/gcc - - - name: Set G++ {{ required_gcc_version }} as default - ansible.builtin.alternatives: - name: g++ - path: "/usr/bin/g++-{{ required_gcc_version }}" - priority: "{{ required_gcc_version }}" - link: /usr/bin/g++ - - - name: Verify GCC version after update - ansible.builtin.shell: gcc --version | head -1 - register: updated_gcc_version - changed_when: false - - - name: Debug updated GCC version - ansible.builtin.debug: - msg: "Updated GCC version: {{ updated_gcc_version.stdout }}" - - - name: Verify G++ version after update - ansible.builtin.shell: g++ --version | head -1 - register: updated_gpp_version - changed_when: false - - - name: Debug updated G++ version - ansible.builtin.debug: - msg: "Updated G++ version: {{ updated_gpp_version.stdout }}" - - - name: Verify alternatives are set correctly - ansible.builtin.shell: update-alternatives --display gcc - register: gcc_alternatives_check - changed_when: false - - - name: Debug GCC alternatives status - ansible.builtin.debug: - msg: "GCC alternatives status: {{ gcc_alternatives_check.stdout }}" - - when: required_gcc_version != current_gcc_major diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/defaults/main.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/defaults/main.yaml deleted file mode 100644 index 15a621b3ad..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/defaults/main.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -secret_project: "{{ project_id }}" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml deleted file mode 100644 index 50eab08ffa..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -# General secret handling -- name: Generate random database password - ansible.builtin.set_fact: - database_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}" - -- name: Generate random webapp admin password - ansible.builtin.set_fact: - webapp_admin_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}" - -- name: Compute Webapp admin password hash to save to Webapp server - ansible.builtin.set_fact: - webapp_admin_hash: "{{ webapp_admin_password | hash('sha256') | upper }}" - -- name: Get access token for Secret Manager API - ansible.builtin.uri: - url: "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" - method: GET - headers: - Metadata-Flavor: Google - return_content: true - register: access_token_result - -- name: Create a GCP secret for webapp server login - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets?secretId=webapp-server-password-{{ deployment_name }}" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - replication: - automatic: {} - status_code: [200, 409] # 409 = already exists - register: webapp_secret_create - -- name: Add webapp password version to secret - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/webapp-server-password-{{ deployment_name }}:addVersion" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - payload: - data: "{{ webapp_admin_password | b64encode }}" - register: webapp_secret - -- name: Create a GCP secret for database password - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets?secretId=db-password-{{ deployment_name }}" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - replication: - automatic: {} - status_code: [200, 409] # 409 = already exists - register: database_secret_create - -- name: Add database password version to secret - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/db-password-{{ deployment_name }}:addVersion" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - payload: - data: "{{ database_password | b64encode }}" - register: database_secret - -# Per-user secret handling -- name: Generate per-user secrets and enrich list - include_tasks: user_secret_tasks.yaml - loop: "{{ vdi_users }}" - loop_control: - loop_var: item - -- name: Set enriched vdi_users_updated variable - ansible.builtin.set_fact: - vdi_users_updated: "{{ vdi_users_updated }}" - when: vdi_users_updated is defined diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml deleted file mode 100644 index 00e7fe540d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -- name: Compute default secret_name if none provided - set_fact: - secret_name: "vdi-user-password-{{ item.username }}-{{ deployment_name }}" - when: - - item.password is not defined - - item.secret_name is not defined - -- name: Determine password source for user {{ item.username }} - ansible.builtin.set_fact: - password_source: >- - {%- if item.secret_name is defined -%}secret - {%- elif item.password is defined -%}password - {%- else -%}generate{%- endif -%} - -- name: Fetch existing Secret Manager password for user {{ item.username }} - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets/{{ item.secret_name }}/versions/latest:access" - method: GET - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - return_content: true - register: sm_result - failed_when: false - when: password_source | trim == "secret" - -- name: Set VDI user password from Secret Manager - ansible.builtin.set_fact: - vdiuser_password: "{{ sm_result.json.payload.data | b64decode }}" - when: password_source == "secret" and sm_result.status == 200 - -- name: Generate random password for user {{ item.username }} - ansible.builtin.set_fact: - vdiuser_password: "{{ lookup('password', '/dev/null length=16 chars=ascii_letters,digits') }}" - when: password_source == "generate" - -- name: Set user password from provided value - ansible.builtin.set_fact: - vdiuser_password: "{{ item.password }}" - when: password_source == "password" - -- name: Create secret for user {{ item.username }} (provided password) - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets?secretId=vdi-user-password-{{ item.username }}-{{ deployment_name }}" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - replication: - automatic: {} - status_code: [200, 409] # 409 = already exists - when: password_source == "password" - -- name: Store provided password in Secret Manager for user {{ item.username }} - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets/vdi-user-password-{{ item.username }}-{{ deployment_name }}:addVersion" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - payload: - data: "{{ item.password | b64encode }}" - when: password_source == "password" - -- name: Create secret for user {{ item.username }} - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets?secretId={{ item.secret_name | default(secret_name) }}" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - replication: - automatic: {} - status_code: [200, 409] # 409 = already exists - when: password_source == "generate" - -- name: Store generated password in Secret Manager for user {{ item.username }} - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets/{{ item.secret_name | default(secret_name) }}:addVersion" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - payload: - data: "{{ vdiuser_password | b64encode }}" - when: password_source == "generate" - -- name: Build enriched user list - ansible.builtin.set_fact: - vdi_users_updated: >- - {{ - (vdi_users_updated | default([])) - + [ - item - | combine({ 'password': vdiuser_password }) - | combine( - (vnc_flavor | lower in ['tightvnc', 'tigervnc']) - | ternary({ - 'vncserver_password': lookup('community.general.random_string', length=8, special=false), - 'display_number': item.port | int - 5900 - }, {}) - ) - ] - }} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml deleted file mode 100644 index 39ddf37c18..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -- name: Provision VDI users (custom or generated passwords) - block: - - # Create the Linux user with a hashed password - - name: Create VDI user {{ item.username }} - ansible.builtin.user: - name: "{{ item.username }}" - groups: "{{ vdi_user_group }},wheel" - shell: /bin/bash - create_home: yes - append: true - password: "{{ item.password | password_hash('sha512') }}" - update_password: on_create # sets pw if user is newly created - - # SSH key setup - - name: Ensure .ssh directory for {{ item.username }} - ansible.builtin.file: - path: "/home/{{ item.username }}/.ssh" - state: directory - owner: "{{ item.username }}" - group: "{{ vdi_user_group }}" - mode: '0700' - - - name: Generate SSH key pair for {{ item.username }} - community.crypto.openssh_keypair: - path: "/home/{{ item.username }}/.ssh/id_rsa" - owner: "{{ item.username }}" - group: "{{ vdi_user_group }}" - mode: '0600' - register: ssh_key - - - name: Add public key to authorized_keys for {{ item.username }} - ansible.builtin.authorized_key: - user: "{{ item.username }}" - key: "{{ ssh_key.public_key }}" - state: present diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml deleted file mode 100644 index 569db3dd6c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -- name: Ensure 'wheel' group is present - group: - name: wheel - state: present - -- name: Ensure VDI user group exists - ansible.builtin.group: - name: "{{ vdi_user_group }}" - state: present - -# Run localā€user provisioning if selected -- name: Provision local users - ansible.builtin.include_tasks: local_users.yaml - loop: "{{ vdi_users_updated }}" - loop_control: - loop_var: item - when: user_provision | lower == 'local_users' - -# Run OS Login tasks if selected -- name: Provision OS Login users - ansible.builtin.include_tasks: os_login.yaml - loop: "{{ vdi_users_updated }}" - loop_control: - loop_var: item - when: user_provision | lower == 'os_login' diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/os_login.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/os_login.yaml deleted file mode 100644 index 3039c31d62..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/user_provision/tasks/os_login.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml deleted file mode 100644 index 4bb770a0c1..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml +++ /dev/null @@ -1,212 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -- name: Ensure PostgreSQL data & init directories exist - ansible.builtin.file: - path: "{{ item }}" - state: directory - owner: root - group: root - mode: '0755' - loop: - - /opt/guacamole-db/data - - /opt/guacamole-db/initdb - -- name: Check Docker service status - ansible.builtin.service_facts: - -- name: Ensure Docker service is running - ansible.builtin.service: - name: docker - state: started - enabled: yes - register: docker_service_status - -- name: Wait for Docker daemon to be ready - ansible.builtin.wait_for: - path: /var/run/docker.sock - timeout: 30 - -- name: Test Docker connectivity - ansible.builtin.command: docker version - register: docker_test - retries: 5 - delay: 2 - until: docker_test.rc == 0 - -- name: Enable pgcrypto extension for initdb - ansible.builtin.copy: - dest: /opt/guacamole-db/initdb/00-enable-pgcrypto.sql - owner: root - group: root - mode: '0644' - content: | - -- Make digest(), hmac(), etc. available - CREATE EXTENSION IF NOT EXISTS pgcrypto; - -- name: Template per-user Guacamole user SQL - ansible.builtin.template: - src: user_bootstrap.sql.j2 - dest: "/opt/guacamole-db/initdb/02-{{ item.username }}_user.sql" - owner: root - group: root - mode: '0644' - loop: "{{ vdi_users_updated }}" - loop_control: - loop_var: item - vars: - username: "{{ item.username }}" - vdi_password: "{{ item.password }}" - -- name: Template per-user SQL for Guacamole connections - ansible.builtin.template: - src: connection_bootstrap.sql.j2 - dest: "/opt/guacamole-db/initdb/03-{{ item.username }}_connection.sql" - owner: root - group: root - mode: '0644' - loop: "{{ vdi_users_updated }}" - loop_control: - loop_var: item - vars: - port: "{{ item.port }}" - username: "{{ item.username }}" - vnc_password: "{{ item.vncserver_password }}" - -- name: Generate combined initdb.sql - ansible.builtin.shell: | - docker run --rm \ - -v /opt/guacamole-db/initdb:/opt/guacamole-db/initdb \ - guacamole/guacamole:latest \ - /opt/guacamole/bin/initdb.sh --postgresql \ - > /opt/guacamole-db/initdb/01-initdb.sql - args: - creates: /opt/guacamole-db/initdb/01-initdb.sql - -- name: Update guacadmin password hash in 01-initdb.sql - ansible.builtin.replace: - path: /opt/guacamole-db/initdb/01-initdb.sql - regexp: "decode\\('CA458A7D494E3BE824F5E1E175A1556C0F8EEF2C2D7DF3633BEC4A29C4411960', 'hex'\\)" - replace: "decode('{{ webapp_admin_hash }}', 'hex')" - -- name: Remove salt from guacadmin in 01-initdb.sql - ansible.builtin.replace: - path: /opt/guacamole-db/initdb/01-initdb.sql - regexp: "decode\\('FE24ADC5E11E2B25288D1704ABE67A79E342ECC26064CE69C5B3177795A82264', 'hex'\\)" - replace: "NULL" - -- name: Pull required Docker images - ansible.builtin.command: docker pull {{ item }} - loop: - - postgres:latest - - guacamole/guacd:latest - - guacamole/guacamole:latest - -- name: Create a dedicated Docker network - ansible.builtin.command: docker network create guac_net - register: network_create - failed_when: - - network_create.rc != 0 - - "'already exists' not in network_create.stderr" - -- name: Start Guacamole PostgreSQL container - ansible.builtin.command: > - docker run -d --name guac_db --network guac_net - -p 5432:5432 - -e POSTGRES_USER=guacamole_db - -e POSTGRES_PASSWORD={{ database_password }} - -e POSTGRES_DB=guacamole_db - -v /opt/guacamole-db/data:/var/lib/postgresql/data - -v /opt/guacamole-db/initdb:/docker-entrypoint-initdb.d - --restart always - postgres:latest - register: postgres_start - failed_when: - - postgres_start.rc != 0 - - "'already in use' not in postgres_start.stderr" - -- name: Wait for PostgreSQL on host port - ansible.builtin.wait_for: - host: 127.0.0.1 - port: 5432 - delay: 2 - timeout: 60 - -- name: Start guacd container - ansible.builtin.command: > - docker run -d --name guacd --network host - --restart always - guacamole/guacd:latest - register: guacd_start - failed_when: - - guacd_start.rc != 0 - - "'already in use' not in guacd_start.stderr" - -- name: Start Guacamole webapp container - ansible.builtin.command: > - docker run -d --name guac_app --network host - -e POSTGRES_HOSTNAME=127.0.0.1 - -e POSTGRES_PORT=5432 - -e POSTGRES_DATABASE=guacamole_db - -e POSTGRES_USER=guacamole_db - -e POSTGRES_PASSWORD={{ database_password }} - -e POSTGRES_AUTO_CREATE_ACCOUNTS=true - -e GUACD_HOSTNAME=localhost - -e GUACD_PORT=4822 - --restart always - guacamole/guacamole:latest - register: guac_app_start - failed_when: - - guac_app_start.rc != 0 - - "'already in use' not in guac_app_start.stderr" - -- name: Wait for Guacamole HTTP endpoint - ansible.builtin.wait_for: - host: 127.0.0.1 - port: "{{ vdi_webapp_port }}" - delay: 2 - timeout: 60 - -- name: Check Guacamole UI is up - ansible.builtin.uri: - url: "http://localhost:{{ vdi_webapp_port }}/guacamole/" - status_code: 200 - return_content: no - register: guac_health - until: guac_health.status == 200 - retries: 10 - delay: 3 - -- name: Obtain Guacamole API token to verify login - ansible.builtin.uri: - url: "http://localhost:{{ vdi_webapp_port }}/guacamole/api/tokens" - method: POST - body_format: form-urlencoded - body: - username: "guacadmin" - password: "{{ webapp_admin_password }}" - return_content: yes - register: guac_api - until: - - guac_api.status == 200 - - guac_api.json.authToken is defined - retries: 10 - delay: 3 - -- name: Remove bootstrap SQL files unless debug is enabled - ansible.builtin.file: - path: /opt/guacamole-db/initdb - state: absent - when: not debug | default(false) diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml deleted file mode 100644 index 4257eb3b52..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -- name: "Install Guacamole" - include_tasks: guacamole.yaml - when: vdi_tool | lower == 'guacamole' - -- name: "Install NoMachine" - include_tasks: nomachine.yaml - when: vdi_tool | lower == 'nomachine' - -- name: "Install Workspot" - include_tasks: workspot.yaml - when: vdi_tool | lower == 'workspot' diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/nomachine.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/nomachine.yaml deleted file mode 100644 index 3039c31d62..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/nomachine.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/workspot.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/workspot.yaml deleted file mode 100644 index 3039c31d62..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/workspot.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/connection_bootstrap.sql.j2 b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/connection_bootstrap.sql.j2 deleted file mode 100644 index e90562fccc..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/connection_bootstrap.sql.j2 +++ /dev/null @@ -1,72 +0,0 @@ --- VNC Connection for {{ username }} -WITH - user_entity AS ( - SELECT entity_id - FROM guacamole_entity - WHERE name = '{{ username }}' AND type = 'USER' - ), - new_connection AS ( - INSERT INTO guacamole_connection (connection_name, protocol) - VALUES ('{{ username }}', 'vnc') - RETURNING connection_id - ), - insert_params AS ( - INSERT INTO guacamole_connection_parameter (connection_id, parameter_name, parameter_value) - SELECT connection_id, param_name, param_value - FROM new_connection, (VALUES - ('hostname', '{{ ansible_default_ipv4.address }}'), - ('port', '{{ port }}'), - ('username', '{{ username }}'), - ('password', '{{ vnc_password }}') - ) AS params(param_name, param_value) - ), - insert_permissions AS ( - INSERT INTO guacamole_connection_permission (entity_id, connection_id, permission) - SELECT - user_entity.entity_id, - new_connection.connection_id, - perms.permission::guacamole_object_permission_type - FROM user_entity, new_connection, - (VALUES ('READ'),('UPDATE'),('DELETE'),('ADMINISTER')) AS perms(permission) - ) -SELECT 1; - --- SSH Connection for {{ username }} -WITH - user_entity AS ( - SELECT entity_id - FROM guacamole_entity - WHERE name = '{{ username }}' AND type = 'USER' - ), - new_connection_ssh AS ( - INSERT INTO guacamole_connection (connection_name, protocol) - VALUES ('{{ username }} SSH', 'ssh') - RETURNING connection_id - ), - insert_params_ssh AS ( - INSERT INTO guacamole_connection_parameter (connection_id, parameter_name, parameter_value) - SELECT connection_id, param_name, param_value - FROM new_connection_ssh, (VALUES - ('hostname', '{{ ansible_default_ipv4.address }}'), - ('port', '22'), - ('username', '{{ username }}'), - ( - 'private-key', - E'{{ lookup("file", "/home/" ~ username ~ "/.ssh/id_rsa") - | replace("\\", "\\\\") - | replace("'", "''") - | replace("\r", "") - | replace("\n", "\\n") }}' - ) - ) AS params(param_name, param_value) - ), - insert_permissions_ssh AS ( - INSERT INTO guacamole_connection_permission (entity_id, connection_id, permission) - SELECT - user_entity.entity_id, - new_connection_ssh.connection_id, - perms.permission::guacamole_object_permission_type - FROM user_entity, new_connection_ssh, - (VALUES ('READ'),('UPDATE'),('DELETE'),('ADMINISTER')) AS perms(permission) - ) -SELECT 1; diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/user_bootstrap.sql.j2 b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/user_bootstrap.sql.j2 deleted file mode 100644 index fb6fc25571..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/user_bootstrap.sql.j2 +++ /dev/null @@ -1,22 +0,0 @@ --- Create Guacamole entity + user for {{ username }} -WITH new_entity AS ( - INSERT INTO guacamole_entity (name, type) - VALUES ('{{ username }}', 'USER') - RETURNING entity_id -) -INSERT INTO guacamole_user ( - entity_id, - password_salt, - password_hash, - password_date, - disabled, - expired -) -SELECT - entity_id, - NULL, - digest('{{ vdi_password }}', 'SHA-256'), - now(), - false, - false -FROM new_entity; diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml deleted file mode 100644 index bc81213782..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml +++ /dev/null @@ -1,23 +0,0 @@ - -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -- name: "Install TigerVNC" - include_tasks: tigervnc.yaml - when: vnc_flavor | lower == 'tigervnc' - -- name: "Install TightVNC" - include_tasks: tightvnc.yaml - when: vnc_flavor | lower == 'tightvnc' diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/tigervnc.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/tigervnc.yaml deleted file mode 100644 index 0401806493..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/tigervnc.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -# Per-user configuration -- name: Configure per-user VNC settings - include_tasks: vnc_user_config.yaml - loop: "{{ vdi_users_updated }}" - loop_control: - loop_var: vdi_user - -# Global VNC users mapping (for systemd service) -- name: Template TigerVNC users file - ansible.builtin.template: - src: vncserver.users.j2 - dest: /etc/tigervnc/vncserver.users - mode: '0644' - vars: - guac_map: "{{ vdi_users_updated }}" - -# Systemd units -- name: Deploy per-user TigerVNC systemd unit (Ubuntu/Debian) - ansible.builtin.template: - src: vncserver@.service.ubuntu.j2 - dest: "/etc/systemd/system/vncserver@{{ item.username }}.service" - mode: '0644' - vars: - display_number: "{{ item.display_number }}" - when: ansible_distribution in ["Ubuntu", "Debian"] - loop: "{{ vdi_users_updated }}" - -- name: Deploy TigerVNC systemd unit (Rocky) - ansible.builtin.template: - src: vncserver@.service.rocky.j2 - dest: /etc/systemd/system/vncserver@.service - mode: '0644' - when: ansible_os_family == "RedHat" - -- name: Reload systemd daemon - ansible.builtin.systemd: - daemon_reload: yes - -# Enable & start each user's VNC server service -- name: Enable & start TigerVNC (Ubuntu/Debian) - systemd: - name: "vncserver@{{ item.username }}" - enabled: yes - state: started - when: ansible_distribution in ["Ubuntu", "Debian"] - loop: "{{ vdi_users_updated }}" - -- name: Enable & start TigerVNC (Rocky) - systemd: - name: "vncserver@:{{ item.display_number }}" - enabled: yes - state: started - when: ansible_os_family == "RedHat" - loop: "{{ vdi_users_updated }}" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/tightvnc.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/tightvnc.yaml deleted file mode 100644 index 3039c31d62..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/tightvnc.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/vnc_user_config.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/vnc_user_config.yaml deleted file mode 100644 index 206551b270..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/tasks/vnc_user_config.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -- name: Ensure VNC configuration directory exists - ansible.builtin.file: - path: "/home/{{ vdi_user.username }}/.vnc" - state: directory - owner: "{{ vdi_user.username }}" - group: "{{ vdi_user_group }}" - mode: '0700' - -- name: Write VNC xstartup for {{ vdi_user.username }} (Ubuntu/Debian) - ansible.builtin.copy: - dest: "/home/{{ vdi_user.username }}/.vnc/xstartup" - content: | - #!/bin/sh - unset SESSION_MANAGER - unset DBUS_SESSION_BUS_ADDRESS - exec startxfce4 - owner: "{{ vdi_user.username }}" - group: "{{ vdi_user_group }}" - mode: '0755' - when: ansible_distribution in ["Ubuntu", "Debian"] - -- name: Write VNC password file for {{ vdi_user.username }} - ansible.builtin.shell: | - echo "{{ vdi_user.vncserver_password }}" | vncpasswd -f > "/home/{{ vdi_user.username }}/.vnc/passwd" - args: - creates: "/home/{{ vdi_user.username }}/.vnc/passwd" - become_user: "{{ vdi_user.username }}" - -- name: Fix passwd file permissions - ansible.builtin.file: - path: "/home/{{ vdi_user.username }}/.vnc/passwd" - owner: "{{ vdi_user.username }}" - group: "{{ vdi_user_group }}" - mode: '0600' - -- name: Copy per-user VNC config - ansible.builtin.template: - src: config.j2 - dest: "/home/{{ vdi_user.username }}/.vnc/config" - owner: "{{ vdi_user.username }}" - group: "{{ vdi_user_group }}" - mode: '0644' - vars: - vdi_resolution: "{{ vdi_resolution }}" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/config.j2 b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/config.j2 deleted file mode 100644 index 605ec72eca..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/config.j2 +++ /dev/null @@ -1,2 +0,0 @@ -session=xfce -geometry={{ vdi_resolution }} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver.users.j2 b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver.users.j2 deleted file mode 100644 index cbdab2dbc0..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver.users.j2 +++ /dev/null @@ -1,3 +0,0 @@ -{% for user in guac_map %} -:{{ user.port - 5900 }}={{ user.username }} -{% endfor %} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.rocky.j2 b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.rocky.j2 deleted file mode 100644 index a7cc3cc10a..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.rocky.j2 +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=TigerVNC server for user %i -After=syslog.target network.target - -[Service] -Type=forking -ExecStart=/usr/libexec/vncsession-start %i -ExecStop=-/bin/true -PIDFile=/run/vncsession-%i.pid - -Restart=always -RestartSec=15 - -[Install] -WantedBy=multi-user.target diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.ubuntu.j2 b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.ubuntu.j2 deleted file mode 100644 index 1df046f446..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.ubuntu.j2 +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=TigerVNC server for user %i -After=network.target - -[Service] -Type=forking -User=%i -Environment=HOME=/home/%i -ExecStart=/usr/bin/vncserver -localhost no :{{ display_number }} -ExecStop=/usr/bin/vncserver -kill :{{ display_number }} - -Restart=on-failure -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl deleted file mode 100644 index eed3983fb0..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by Cluster Toolkit - -- hosts: localhost - become: true - vars_files: - - "/tmp/vdi/vars.yaml" - tasks: - -%{ for r in roles } - - import_role: - name: "${r}" -%{ endfor } diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl deleted file mode 100644 index 9960ecf9db..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Cluster Toolkit - -deployment_name: ${deployment_name} -project_id: ${project_id} -user_provision: ${user_provision} -vnc_flavor: ${vnc_flavor} -vdi_tool: ${vdi_tool} -vdi_user_group: ${vdi_user_group} -vdi_webapp_port: ${vdi_webapp_port} -vdi_resolution: ${vdi_resolution} - -vdi_users: -%{ for user in vdi_users } - - username: ${user.username} - port: ${user.port} -%{ if user.secret_name != null } - secret_name: "${user.secret_name}" -%{ endif } -%{ if user.secret_project != null } - secret_project: "${user.secret_project}" -%{ endif } - -%{ endfor } diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/validation.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/validation.tf deleted file mode 100644 index 02c3dfc822..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/validation.tf +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resource "terraform_data" "input_validation" { - lifecycle { - precondition { - condition = contains(["tigervnc"], var.vnc_flavor) - error_message = "vnc_flavor must be tigervnc." - } - - precondition { - condition = contains(["guacamole"], var.vdi_tool) - error_message = "vdi_tool must be one of: guacamole." - } - - precondition { - condition = contains(["local_users"], var.user_provision) - error_message = "user_provision must be local_users." - } - - precondition { - condition = can(regex("^[1-9][0-9]*x[1-9][0-9]*$", var.vdi_resolution)) - error_message = "vdi_resolution must be in the form WIDTHxHEIGHT (e.g. 1920x1080)." - } - - precondition { - condition = var.user_provision == "local_users" || length(var.vdi_users) == 0 - error_message = "vdi_users may only be set when user_provision = local_users." - } - - precondition { - condition = ( - var.vnc_flavor != "tigervnc" && var.vnc_flavor != "tightvnc" - ) || alltrue([ - for user in var.vdi_users : ( - user.port >= var.vnc_port_min && user.port <= var.vnc_port_max - ) - ]) - error_message = "Each VDI user must have a port between 5901 and 5999 when VNC is used." - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/variables.tf deleted file mode 100644 index 38cf7bd42a..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/variables.tf +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "project_id" { - description = "Project in which the HPC deployment will be created." - type = string -} - -variable "deployment_name" { - description = "The name of the deployment." - type = string -} - -variable "region" { - description = "Region to place bucket containing startup script." - type = string -} - -variable "labels" { - description = "Key-value pairs of labels to be added to created resources." - type = map(string) -} - -variable "vnc_flavor" { - description = "The VNC server flavor to use (tigervnc currently supported)" - type = string - default = "tigervnc" -} - -variable "vdi_tool" { - type = string - description = "VDI tool to deploy (guacamole currently supported)." - default = "guacamole" -} - -variable "user_provision" { - type = string - description = "User type to create (local_users supported. os-login to do." - default = "local_users" -} - -variable "vdi_user_group" { - type = string - description = "Unix group to create/use for VDI users." - default = "vdiusers" -} - -variable "vdi_resolution" { - type = string - description = "Desktop resolution for VNC sessions (e.g. 1920x1080)." - default = "1920x1080" -} - -variable "vdi_webapp_port" { - type = string - description = "Port to serve the Webapp interface from if applicable" - default = "8080" -} - -variable "vdi_users" { - description = "List of VDI users to configure. Passwords are handled securely by the Ansible roles: if secret_name is provided, the password is fetched from Secret Manager; if neither password nor secret_name is provided, a random password is generated and stored in Secret Manager. If secret_project is provided, it specifies the GCP project where the secret is stored (defaults to the deployment project)." - type = list(object({ - username = string - port = number - secret_name = optional(string) - secret_project = optional(string) - })) - default = [] -} - -variable "vnc_port_min" { - type = number - default = 5901 - description = "Minimum valid VNC port." -} -variable "vnc_port_max" { - type = number - default = 5999 - description = "Maximum valid VNC port." -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/versions.tf deleted file mode 100644 index 6024b738a7..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/vdi-setup/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 1.0" - required_providers { - archive = { - source = "hashicorp/archive" - version = "~> 2.0" - } - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/README.md deleted file mode 100644 index ee9c057c39..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/README.md +++ /dev/null @@ -1,87 +0,0 @@ -## Description - -This module will insert a dependency on the completion of the startup script -for one or more specified compute VMs and report back if it fails. This can be useful when running -post-boot installation scripts that require the startup script to finish setting up a node. - -> **_WARNING:_**: this module is experimental and not fully supported. - -### Additional Dependencies - -* [**gcloud**](https://cloud.google.com/sdk/gcloud) must be present in the path - of the machine where `terraform apply` is run. - -### Example - -```yaml -- id: workstation - source: modules/compute/vm-instance - use: - - network1 - - my-startup-script - settings: - instance_count: 4 - -# Wait for all instances of the above VM to finish running startup scripts. -- id: wait - source: community/modules/scripts/wait-for-startup - settings: - instance_names: $(workstation.name) -``` - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [null](#requirement\_null) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [null](#provider\_null) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [null_resource.validate_instance_names](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.wait_for_startup](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [gcloud\_path\_override](#input\_gcloud\_path\_override) | Directory of the gcloud executable to be used during cleanup | `string` | `""` | no | -| [instance\_name](#input\_instance\_name) | Name of the instance we are waiting for (can be null if 'instance\_names' is not empty) | `string` | `null` | no | -| [instance\_names](#input\_instance\_names) | A list of instance names we are waiting for, in addition to the one mentioned in 'instance\_name' (if any) | `list(string)` | `[]` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [timeout](#input\_timeout) | Timeout in seconds | `number` | `1200` | no | -| [zone](#input\_zone) | The GCP zone where the instance is running | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/main.tf deleted file mode 100644 index 3f6b416251..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/main.tf +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - combined_instance_names = concat(var.instance_names, [var.instance_name]) -} - -resource "null_resource" "validate_instance_names" { - lifecycle { - precondition { - condition = var.instance_name != null || length(var.instance_names) > 0 - error_message = "At least one instance name must be provided" - } - } -} - -resource "null_resource" "wait_for_startup" { - count = length(local.combined_instance_names) - - provisioner "local-exec" { - command = "/bin/bash ${path.module}/scripts/wait-for-startup-status.sh" - environment = { - INSTANCE_NAME = self.triggers.instance_name - ZONE = var.zone - PROJECT_ID = var.project_id - TIMEOUT = var.timeout - GCLOUD_PATH = var.gcloud_path_override - } - } - - triggers = { - instance_name = local.combined_instance_names[count.index] - } -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf deleted file mode 100644 index 11a2ddf118..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/outputs.tf +++ /dev/null @@ -1,15 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh deleted file mode 100644 index 4a231f7def..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/scripts/wait-for-startup-status.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [[ -z "${INSTANCE_NAME}" ]]; then - echo "INSTANCE_NAME is unset... exiting" - exit 0 -fi -if [[ -z "${ZONE}" ]]; then - echo "ZONE is unset" - exit 1 -fi -if [[ -z "${PROJECT_ID}" ]]; then - echo "PROJECT_ID is unset" - exit 1 -fi -if [[ -z "${TIMEOUT}" ]]; then - echo "TIMEOUT is unset" - exit 1 -fi - -if [[ -n "${GCLOUD_PATH}" ]]; then - export PATH="$GCLOUD_PATH:$PATH" -fi - -echo "Waiting for startup: instance_name='${INSTANCE_NAME}', zone='${ZONE}', project_id='${PROJECT_ID}', timeout_seconds='${TIMEOUT}'" - -# Wrapper around grep that swallows the error status code 1 -c1grep() { grep "$@" || test $? = 1; } - -now=$(date +%s) - -# If VM was created more than 30 days ago, serial port logs may no longer exist. -# Exit without errors if the instance is older than 30 days. -logsExpiryDays=30 -createdTimestampIso=$(gcloud compute instances describe "${INSTANCE_NAME}" --project "${PROJECT_ID}" --zone "${ZONE}" --format "value(creationTimestamp)") -earliestAllowedCreatedTimestamp=$(date -d "${createdTimestampIso} +${logsExpiryDays} day" +%s) -if [[ "$earliestAllowedCreatedTimestamp" -lt "$now" ]]; then - echo "Instance was created more than 30 days ago - serial port 1 logs are likely expired... exiting" - exit 0 -fi - -deadline=$((now + TIMEOUT)) -error_file=$(mktemp) -fetch_cmd="gcloud compute instances get-serial-port-output ${INSTANCE_NAME} --port 1 --zone ${ZONE} --project ${PROJECT_ID}" -# Match string for all finish types of the old guest agent and successful -# finishes on the new guest agent -FINISH_LINE="startup-script exit status" -# Match string for failures on the new guest agent -FINISH_LINE_ERR="Script.*failed with error:" - -NON_FATAL_ERRORS=( - "Internal error" -) - -until [[ now -gt deadline ]]; do - ser_log=$( - set -o pipefail - ${fetch_cmd} 2>"${error_file}" | - c1grep "${FINISH_LINE}\|${FINISH_LINE_ERR}" - ) || { - err=$(cat "${error_file}") - echo "$err" - fatal_error="true" - for e in "${NON_FATAL_ERRORS[@]}"; do - if [[ $err = *"$e"* ]]; then - fatal_error="false" - break - fi - done - - if [[ $fatal_error = "true" ]]; then - exit 1 - fi - } - if [[ -n "${ser_log}" ]]; then break; fi - echo "Could not detect end of startup script. Sleeping." - sleep 5 - now=$(date +%s) -done - -# This line checks for an exit code - the assumption is that there is a number -# at the end of the line and it is an exit code -STATUS=$(sed -r 's/.*([0-9]+)\s*$/\1/' <<<"${ser_log}" | uniq) -# This specific text is monitored for in tests, do not change. -INSPECT_OUTPUT_TEXT="To inspect the startup script output, please run:" -if [[ "${STATUS}" == 0 ]]; then - echo "startup-script finished successfully" -elif [[ "${STATUS}" == 1 ]]; then - echo "startup-script finished with errors, ${INSPECT_OUTPUT_TEXT}" - echo "${fetch_cmd}" -elif [[ now -ge deadline ]]; then - echo "startup-script timed out after ${TIMEOUT} seconds" - echo "${INSPECT_OUTPUT_TEXT}" - echo "${fetch_cmd}" - exit 1 -else - echo "Invalid return status: '${STATUS}'" - echo "${INSPECT_OUTPUT_TEXT}" - echo "${fetch_cmd}" - exit 1 -fi - -exit "${STATUS}" diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf deleted file mode 100644 index fe6410a920..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/variables.tf +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "instance_name" { - description = "Name of the instance we are waiting for (can be null if 'instance_names' is not empty)" - type = string - default = null -} - -variable "instance_names" { - description = "A list of instance names we are waiting for, in addition to the one mentioned in 'instance_name' (if any)" - type = list(string) - default = [] -} - -variable "zone" { - description = "The GCP zone where the instance is running" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "timeout" { - description = "Timeout in seconds" - type = number - default = 1200 - validation { - condition = var.timeout >= 0 - error_message = "The timeout should be non-negative" - } -} - -variable "gcloud_path_override" { - description = "Directory of the gcloud executable to be used during cleanup" - type = string - default = "" - nullable = false -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf deleted file mode 100644 index bb5a32e08d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/wait-for-startup/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:wait-for-startup/v1.57.0" - } - - required_version = ">= 0.14.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/README.md b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/README.md deleted file mode 100644 index fc25bc0a55..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/README.md +++ /dev/null @@ -1,109 +0,0 @@ -## Description - -This module contains a set of scripts to be used in customizing Windows VMs at -boot or during image building. Please note that the installation of NVIDIA GPU -drivers takes, at minimum, 30-60 minutes. It is therefore recommended to build -a custom image and reuse it as shown below, rather than install GPU drivers at -boot time. - -> NOTE: the output `windows_startup_ps1` must be passed explicitly as shown -> below when used with Packer modules. This is due to a limitation in the `use` -> keyword and inputs of type `list` in Packer modules; this does not impact -> Terraform modules - -### NVIDIA Drivers and CUDA Toolkit - -Many Google Cloud VM families include or can have NVIDIA GPUs attached to them. -This module supports GPU applications by enabling you to easily install -a compatible release of NVIDIA drivers and of the CUDA Toolkit. The script is -the [solution recommended by our documentation][docs] and is [directly sourced -from GitHub][script-src]. - -[docs]: https://cloud.google.com/compute/docs/gpus/install-drivers-gpu#windows -[script-src]: https://github.com/GoogleCloudPlatform/compute-gpu-installation/blob/24dac3004360e0696c49560f2da2cd60fcb80107/windows/install_gpu_driver.ps1 - -```yaml -- group: primary - modules: - - id: network1 - source: modules/network/vpc - settings: - enable_iap_rdp_ingress: true - enable_iap_winrm_ingress: true - - - id: windows_startup - source: community/modules/scripts/windows-startup-script - settings: - install_nvidia_driver: true - -- group: packer - modules: - - id: image - source: modules/packer/custom-image - kind: packer - use: - - network1 - - windows_startup - settings: - source_image_family: windows-2016 - machine_type: n1-standard-8 - accelerator_count: 1 - accelerator_type: nvidia-tesla-t4 - disk_size: 75 - disk_type: pd-ssd - omit_external_ip: false - state_timeout: 15m -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [http\_proxy](#input\_http\_proxy) | Set http and https proxy for use by Invoke-WebRequest commands | `string` | `""` | no | -| [http\_proxy\_set\_environment](#input\_http\_proxy\_set\_environment) | Set system default environment variables http\_proxy and https\_proxy for all commands | `bool` | `false` | no | -| [install\_nvidia\_driver](#input\_install\_nvidia\_driver) | Install NVIDIA GPU drivers and the CUDA Toolkit using script specified by var.install\_nvidia\_driver\_script | `bool` | `false` | no | -| [install\_nvidia\_driver\_args](#input\_install\_nvidia\_driver\_args) | Arguments to supply to NVIDIA driver install script | `string` | `"/s /n"` | no | -| [install\_nvidia\_driver\_script](#input\_install\_nvidia\_driver\_script) | Install script for NVIDIA drivers specified by http/https URL | `string` | `"https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda_12.1.1_531.14_windows.exe"` | no | -| [no\_proxy](#input\_no\_proxy) | Environment variables no\_proxy (only used if var.http\_proxy\_set\_environment is enabled) | `string` | `"169.254.169.254,metadata,metadata.google.internal,.googleapis.com"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [windows\_startup\_ps1](#output\_windows\_startup\_ps1) | A string list of scripts selected by this module | - diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/main.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/main.tf deleted file mode 100644 index 5e6bc8b94d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/main.tf +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - setx_http_proxy_ps1 = !var.http_proxy_set_environment ? [] : [ - templatefile("${path.module}/templates/setx_http_proxy.ps1", { - "http_proxy" : var.http_proxy, - "no_proxy" : var.no_proxy, - }) - ] - - nvidia_ps1 = !var.install_nvidia_driver ? [] : [ - templatefile("${path.module}/templates/install_gpu_driver.ps1.tftpl", { - "url" : var.install_nvidia_driver_script - "args" : var.install_nvidia_driver_args - "http_proxy" : var.http_proxy, - }) - ] - - startup_ps1 = concat(local.setx_http_proxy_ps1, local.nvidia_ps1) -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf deleted file mode 100644 index 006ea312ad..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/outputs.tf +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "windows_startup_ps1" { - description = "A string list of scripts selected by this module" - value = local.startup_ps1 -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl deleted file mode 100644 index 55c4a2a3cd..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/install_gpu_driver.ps1.tftpl +++ /dev/null @@ -1,38 +0,0 @@ -#Requires -RunAsAdministrator - -# Windows 2016 needs forced upgrade to TLS 1.2 -[Net.ServicePointManager]::SecurityProtocol = 'Tls12' - -# important for catching exception in Invoke-WebRequest -Set-StrictMode -Version latest -$ErrorActionPreference = 'Stop' - -%{ if http_proxy != "" } -[System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy("${http_proxy}") -%{ endif } - -# Create the folder for the driver download -$file_dir = 'C:\NVIDIA-Driver\nvidia_installer_windows.exe' -if (!(Test-Path -Path 'C:\NVIDIA-Driver')) { - New-Item -Path 'C:\' -Name 'NVIDIA-Driver' -ItemType 'directory' | Out-Null -} - -# Download the file to a specified directory -Write-Output "Downloading ${url} to $file_dir" -# Disabling progress bar has surprising large (10-100x) impact on speed -$ProgressPreference = 'SilentlyContinue' -try { - Invoke-WebRequest -Uri "${url}" -OutFile "$file_dir" -} catch { - Write-Output "$_" - throw "Failed to download ${url}; exiting startup script" -} - -# Install the file with the specified path from earlier as well as the RunAs admin option -Write-Output "Executing $file_dir with arguments '${args}'" -try { - Start-Process -FilePath "$file_dir" -ArgumentList '${args}' -Wait -} catch { - Write-Output "$_" - throw "Could not install NVIDIA driver; exiting startup script" -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 deleted file mode 100644 index e6d84b0681..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/templates/setx_http_proxy.ps1 +++ /dev/null @@ -1,5 +0,0 @@ -#Requires -RunAsAdministrator - -setx http_proxy ${http_proxy} /m -setx https_proxy ${http_proxy} /m -setx no_proxy ${no_proxy} /m diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf deleted file mode 100644 index 9e4fb9e67d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/variables.tf +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "install_nvidia_driver" { - description = "Install NVIDIA GPU drivers and the CUDA Toolkit using script specified by var.install_nvidia_driver_script" - type = bool - default = false -} - -variable "install_nvidia_driver_script" { - description = "Install script for NVIDIA drivers specified by http/https URL" - type = string - default = "https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda_12.1.1_531.14_windows.exe" -} - -variable "install_nvidia_driver_args" { - description = "Arguments to supply to NVIDIA driver install script" - type = string - default = "/s /n" -} - -variable "http_proxy" { - description = "Set http and https proxy for use by Invoke-WebRequest commands" - type = string - default = "" - nullable = false -} - -variable "http_proxy_set_environment" { - description = "Set system default environment variables http_proxy and https_proxy for all commands" - type = bool - default = false - nullable = false -} - -variable "no_proxy" { - description = "Environment variables no_proxy (only used if var.http_proxy_set_environment is enabled)" - type = string - default = "169.254.169.254,metadata,metadata.google.internal,.googleapis.com" - nullable = false -} diff --git a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf b/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf deleted file mode 100644 index 9d5f6bd68d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/community/modules/scripts/windows-startup-script/versions.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:windows-startup-script/v1.57.0" - } - - required_version = ">= 0.14.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/README.md b/vdi-test-scott/primary/modules/embedded/modules/README.md deleted file mode 100644 index 37831fc708..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/README.md +++ /dev/null @@ -1,550 +0,0 @@ -# Modules - -This directory contains a set of core modules built for the Cluster Toolkit. Modules -describe the building blocks of an AI/ML and HPC deployment. The expected fields in a -module are listed in more detail [below](#module-fields). Blueprints can be -extended in functionality by incorporating [modules from GitHub -repositories][ghmods]. - -[ghmods]: #github-modules - -## All Modules - -Modules from various sources are all listed here for visibility. Badges are used -to indicate the source and status of many of these resources. - -Modules listed below with the ![core-badge] badge are located in this -folder and are tested and maintained by the Cluster Toolkit team. - -Modules labeled with the ![community-badge] badge are contributed by -the community (including the Cluster Toolkit team, partners, etc.). Community modules -are located in the [community folder](../community/modules/README.md). - -Modules labeled with the ![deprecated-badge] badge are now deprecated and may be -removed in the future. Customers are advised to transition to alternatives. - -Modules that are still in development and less stable are labeled with the -![experimental-badge] badge. - -[core-badge]: https://img.shields.io/badge/-core-blue?style=plastic -[community-badge]: https://img.shields.io/badge/-community-%23b8def4?style=plastic -[stable-badge]: https://img.shields.io/badge/-stable-lightgrey?style=plastic -[experimental-badge]: https://img.shields.io/badge/-experimental-%23febfa2?style=plastic -[deprecated-badge]: https://img.shields.io/badge/-deprecated-%23fea2a2?style=plastic - -### Compute - -* **[vm-instance]** ![core-badge] : Creates one or more VM instances. -* **[schedmd-slurm-gcp-v6-partition]** ![core-badge] : - Creates a partition to be used by a [slurm-controller][schedmd-slurm-gcp-v6-controller]. -* **[schedmd-slurm-gcp-v6-nodeset]** ![core-badge] : - Creates a nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module. -* **[schedmd-slurm-gcp-v6-nodeset-tpu]** ![core-badge] : - Creates a TPU nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module. -* **[schedmd-slurm-gcp-v6-nodeset-dynamic]** ![core-badge] ![experimental-badge]: - Creates a dynamic nodeset to be used by the [schedmd-slurm-gcp-v6-partition] module and instance template. -* **[gke-node-pool]** ![core-badge] ![experimental-badge] : Creates a - Kubernetes node pool using GKE. -* **[resource-policy]** ![core-badge] ![experimental-badge] : Create a resource policy for compute engines that can be applied to gke-node-pool's nodes. -* **[gke-job-template]** ![core-badge] ![experimental-badge] : Creates a - Kubernetes job file to be used with a [gke-node-pool]. -* **[htcondor-execute-point]** ![community-badge] ![experimental-badge] : - Manages a group of execute points for use in an [HTCondor - pool][htcondor-setup]. -* **[mig]** ![community-badge] ![experimental-badge] : Creates a Managed Instance Group. -* **[notebook]** ![community-badge] ![experimental-badge] : Creates a Vertex AI - Notebook. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. - -[vm-instance]: compute/vm-instance/README.md -[gke-node-pool]: ../modules/compute/gke-node-pool/README.md -[resource-policy]: ../modules/compute/resource-policy/README.md -[gke-job-template]: ../modules/compute/gke-job-template/README.md -[schedmd-slurm-gcp-v6-partition]: ../community/modules/compute/schedmd-slurm-gcp-v6-partition/README.md -[schedmd-slurm-gcp-v6-nodeset]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset/README.md -[schedmd-slurm-gcp-v6-nodeset-tpu]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset-tpu/README.md -[schedmd-slurm-gcp-v6-nodeset-dynamic]: ../community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/README.md -[htcondor-execute-point]: ../community/modules/compute/htcondor-execute-point/README.md -[mig]: ../community/modules/compute/mig/README.md -[notebook]: ../community/modules/compute/notebook/README.md -[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md - -### Database - -* **[slurm-cloudsql-federation]** ![community-badge] ![experimental-badge] : - Creates a [Google SQL Instance](https://cloud.google.com/sql/) meant to be - integrated with a [slurm-controller][schedmd-slurm-gcp-v6-controller]. -* **[bigquery-dataset]** ![community-badge] ![experimental-badge] : Creates a BQ - dataset. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. -* **[bigquery-table]** ![community-badge] ![experimental-badge] : Creates a BQ - table. Primarily used for - [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. - -[slurm-cloudsql-federation]: ../community/modules/database/slurm-cloudsql-federation/README.md -[bigquery-dataset]: ../community/modules/database/bigquery-dataset/README.md -[bigquery-table]: ../community/modules/database/bigquery-table/README.md -[fsi-montecarlo-on-batch]: ../community/modules/files/fsi-montecarlo-on-batch/README.md - -### File System - -* **[filestore]** ![core-badge] : Creates a - [filestore](https://cloud.google.com/filestore) file system. -* **[parallelstore]** ![core-badge] ![experimental-badge]: Creates a - [parallelstore](https://cloud.google.com/parallelstore) file system. -* **[pre-existing-network-storage]** ![core-badge] : Specifies a - pre-existing file system that can be mounted on a VM. -* **[managed-lustre]** ![core-badge] ![experimental-badge]: Creates a - [managed-lustred](https://cloud.google.com/managed-lustre) file system. -* **[DDN-EXAScaler]** ![community-badge] ![deprecated-badge] : Creates - a [DDN EXAscaler lustre](https://www.ddn.com/partners/google-cloud-platform/) - file system. This module is deprecated and will be removed by July 1, 2025. Consider migrating to managed-lustre. -* **[cloud-storage-bucket]** ![community-badge] ![experimental-badge] : Creates - a Google Cloud Storage (GCS) bucket. -* **[gke-persistent-volume]** ![core-badge] ![experimental-badge] : Creates - persistent volumes and persistent volume claims for shared storage. -* **[nfs-server]** ![community-badge] ![experimental-badge] : Creates a VM and - configures an NFS server that can be mounted by other VM. - -[filestore]: file-system/filestore/README.md -[parallelstore]: file-system/parallelstore/README.md -[pre-existing-network-storage]: file-system/pre-existing-network-storage/README.md -[managed-lustre]: file-system/managed-lustre/README.md -[ddn-exascaler]: ../community/modules/file-system/DDN-EXAScaler/README.md -[nfs-server]: ../community/modules/file-system/nfs-server/README.md -[cloud-storage-bucket]: ../community/modules/file-system/cloud-storage-bucket/README.md -[gke-persistent-volume]: ../modules/file-system/gke-persistent-volume/README.md - -### Monitoring - -* **[dashboard]** ![core-badge] : Creates a - [monitoring dashboard](https://cloud.google.com/monitoring/dashboards) for - visually tracking a Cluster Toolkit deployment. - -[dashboard]: monitoring/dashboard/README.md - -### Network - -* **[vpc]** ![core-badge] : Creates a - [Virtual Private Cloud (VPC)](https://cloud.google.com/vpc) network with - regional subnetworks and firewall rules. -* **[multivpc]** ![core-badge] ![experimental-badge]: Creates a variable - number of VPC networks using the [vpc] module. -* **[pre-existing-vpc]** ![core-badge] : Used to connect newly - built components to a pre-existing VPC network. -* **[firewall-rules]** ![core-badge] ![experimental-badge] : Add custom firewall - rules to existing networks (commonly used with [pre-existing-vpc]). -* **[private-service-access]** ![community-badge] ![experimental-badge] : - Configures Private Services Access for a VPC network (commonly used with [filestore] and [slurm-cloudsql-federation]). - -[vpc]: network/vpc/README.md -[multivpc]: network/multivpc/README.md -[pre-existing-vpc]: network/pre-existing-vpc/README.md -[firewall-rules]: network/firewall-rules/README.md -[private-service-access]: ../community/modules/network/private-service-access/README.md - -### Packer - -* **[custom-image]** ![core-badge] : Creates a custom VM Image - based on the GCP HPC VM image. - -[custom-image]: packer/custom-image/README.md - -### Project - -* **[service-account]** ![community-badge] ![experimental-badge] : Creates [service - accounts](https://cloud.google.com/iam/docs/service-accounts) for a GCP - project. -* **[service-enablement]** ![community-badge] ![experimental-badge] : Allows enabling - various APIs for a Google Cloud Project. - -[service-account]: ../community/modules/project/service-account/README.md -[service-enablement]: ../community/modules/project/service-enablement/README.md - -### Pub/Sub - -* **[topic]** ![community-badge] ![experimental-badge] : Creates a -Pub/Sub topic. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. -* **[bigquery-sub]** ![community-badge] ![experimental-badge] : Creates a -Pub/Sub subscription. Primarily used for [FSI - MonteCarlo Tutorial][fsi-montecarlo-on-batch-tutorial]. - -[topic]: ../community/modules/pubsub/topic/README.md -[bigquery-sub]: ../community/modules/pubsub/bigquery-sub/README.md - -### Remote Desktop - -* **[chrome-remote-desktop]** ![community-badge] ![experimental-badge] : Creates - a GPU accelerated Chrome Remote Desktop. - -[chrome-remote-desktop]: ../community/modules/remote-desktop/chrome-remote-desktop/README.md - -### Scheduler - -* **[batch-job-template]** ![core-badge] : Creates a Google Cloud Batch job - template that works with other Toolkit modules. -* **[batch-login-node]** ![core-badge] : Creates a VM that can be used for - submission of Google Cloud Batch jobs. -* **[gke-cluster]** ![core-badge] ![experimental-badge] : Creates a - Kubernetes cluster using GKE. -* **[pre-existing-gke-cluster]** ![core-badge] ![experimental-badge] : Retrieves an existing GKE cluster. Substitute for ([gke-cluster]) module. -* **[schedmd-slurm-gcp-v6-controller]** ![core-badge] : - Creates a Slurm controller node. -* **[schedmd-slurm-gcp-v6-login]** ![core-badge] : - Creates a Slurm login node. -* **[htcondor-setup]** ![community-badge] ![experimental-badge] : Creates the - base infrastructure for an HTCondor pool (service accounts and Cloud Storage bucket). -* **[htcondor-pool-secrets]** ![community-badge] ![experimental-badge] : Creates - and manages access to the secrets necessary for secure operation of an - HTCondor pool. -* **[htcondor-access-point]** ![community-badge] ![experimental-badge] : Creates - a regional instance group managing a highly available HTCondor access point - (login node). - -[batch-job-template]: ../modules/scheduler/batch-job-template/README.md -[batch-login-node]: ../modules/scheduler/batch-login-node/README.md -[gke-cluster]: ../modules/scheduler/gke-cluster/README.md -[pre-existing-gke-cluster]: ../modules/scheduler/pre-existing-gke-cluster/README.md -[htcondor-setup]: ../community/modules/scheduler/htcondor-setup/README.md -[htcondor-pool-secrets]: ../community/modules/scheduler/htcondor-pool-secrets/README.md -[htcondor-access-point]: ../community/modules/scheduler/htcondor-access-point/README.md -[schedmd-slurm-gcp-v6-controller]: ../community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md -[schedmd-slurm-gcp-v6-login]: ../community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md - -### Scripts - -* **[startup-script]** ![core-badge] : Creates a customizable startup script - that can be fed into compute VMs. -* **[windows-startup-script]** ![community-badge] ![experimental-badge]: Creates - Windows PowerShell (PS1) scripts that can be used to customize Windows VMs - and VM images. -* **[htcondor-install]** ![community-badge] ![experimental-badge] : Creates - a startup script to install HTCondor and exports a list of required APIs -* **[ramble-execute]** ![community-badge] ![experimental-badge] : Creates a - startup script to execute - [Ramble](https://github.com/GoogleCloudPlatform/ramble) commands on a target - VM -* **[ramble-setup]** ![community-badge] ![experimental-badge] : Creates a - startup script to install - [Ramble](https://github.com/GoogleCloudPlatform/ramble) on an instance or a - slurm login or controller. -* **[spack-setup]** ![community-badge] ![experimental-badge] : Creates a startup - script to install [Spack](https://github.com/spack/spack) on an instance or a - slurm login or controller. -* **[spack-execute]** ![community-badge] ![experimental-badge] : Defines a - software build using [Spack](https://github.com/spack/spack). -* **[wait-for-startup]** ![community-badge] ![experimental-badge] : Waits for - successful completion of a startup script on a compute VM. - -[startup-script]: scripts/startup-script/README.md -[windows-startup-script]: ../community/modules/scripts/windows-startup-script/README.md -[htcondor-install]: ../community/modules/scripts/htcondor-install/README.md -[kubernetes-operations]: ../community/modules/scripts/kubernetes-operations/README.md -[ramble-execute]: ../community/modules/scripts/ramble-execute/README.md -[ramble-setup]: ../community/modules/scripts/ramble-setup/README.md -[spack-setup]: ../community/modules/scripts/spack-setup/README.md -[spack-execute]: ../community/modules/scripts/spack-execute/README.md -[wait-for-startup]: ../community/modules/scripts/wait-for-startup/README.md - -## Module Fields - -### ID (Required) - -The `id` field is used to uniquely identify and reference a defined module. -ID's are used in [variables](../examples/README.md#variables) and become the -name of each module when writing the terraform `main.tf` file. They are also -used in the [use](#use-optional) and [outputs](#outputs-optional) lists -described below. - -For terraform modules, the ID will be rendered into the terraform module label -at the top level main.tf file. - -### Source (Required) - -The source is a path or URL that points to the source files for Packer or -Terraform modules. A source can either be a filesystem path or a URL to a git -repository: - -* Filesystem paths - * modules embedded in the `gcluster` executable - * modules in the local filesystem -* Remote modules using [Terraform URL syntax](https://developer.hashicorp.com/terraform/language/modules/sources) - * Hosted on [GitHub](https://developer.hashicorp.com/terraform/language/modules/sources#github) - * Google Cloud Storage [Buckets](https://developer.hashicorp.com/terraform/language/modules/sources#gcs-bucket) - * Generic [git repositories](https://developer.hashicorp.com/terraform/language/modules/sources#generic-git-repository) - - when modules are in a subdirectory of the git repository, a special - double-slash `//` notation can be required as described below - -An important distinction is that those URLs are natively supported by Terraform so -they are not copied to your deployment directory. Packer does not have native -support for git-hosted modules so the Toolkit will copy these modules into the -deployment folder on your behalf. - -#### Embedded Modules - -Embedded modules are added to the gcluster binary during compilation and cannot -be edited. To refer to embedded modules, set the source path to -`modules/<>` or `community/modules/<>`. - -The paths match the modules in the repository structure for [core modules](./) -and [community modules](../community/modules/). Because the modules are embedded -during compilation, your local copies may differ unless you recompile gcluster. - -For example, this example snippet uses the embedded pre-existing-vpc module: - -```yaml - - id: network1 - source: modules/network/pre-existing-vpc -``` - -#### Local Modules - -Local modules point to a module in the file system and can easily be edited. -They are very useful during module development. To use a local module, set -the source to a path starting with `/`, `./`, or `../`. For instance, the -following module definition refers the local pre-existing-vpc modules. - -```yaml - - id: network1 - source: modules/network/pre-existing-vpc -``` - -> **_NOTE:_** Relative paths (beginning with `.` or `..` must be relative to the -> working directory from which `gcluster` is executed. This example would have to be -> run from a local copy of the Cluster Toolkit repository. An alternative is to use -> absolute paths to modules. - -#### GitHub-hosted Modules and Packages - -To use a Terraform module available on GitHub, set the source to a path starting -with `github.com` (HTTPS) or `git@github.com` (SSH). For instance, the following -module definition sources the Toolkit vpc module: - -```yaml - - id: network1 - source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/network/vpc -``` - -This example uses the [double-slash notation][tfsubdir] (`//`) to indicate that -the Toolkit is a "package" of multiple modules whose root directory is the root -of the git repository. The remainder of the path indicates the sub-directory of -the vpc module. - -The example above uses the default `main` branch of the Toolkit. Specific -[revisions][tfrev] can be selected with any valid [git reference][gitref]. -(git branch, commit hash or tag). If the git reference is a tag or branch, we -recommend setting `&depth=1` to reduce the data transferred over the network. -This option cannot be set when the reference is a commit hash. The following -examples select the vpc module on the active `develop` branch and also an older -release of the filestore module: - -```yaml - - id: network1 - source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/network/vpc?ref=develop - ... - - id: homefs - source: github.com/GoogleCloudPlatform/hpc-toolkit//modules/file-system/filestore?ref=v1.22.1&depth=1 -``` - -Because Terraform modules natively support this syntax, gcluster will not copy -GitHub-hosted modules into your deployment folder. Terraform will download them -into a hidden folder when you run `terraform init`. - -[tfrev]: https://www.terraform.io/language/modules/sources#selecting-a-revision -[gitref]: https://git-scm.com/book/en/v2/Git-Tools-Revision-Selection#_single_revisions -[tfsubdir]: https://www.terraform.io/language/modules/sources#modules-in-package-sub-directories - -##### GitHub-hosted Packer modules - -Packer does not natively support GitHub-hosted modules so `gcluster create` will -copy modules into your deployment folder. - -If the module uses `//` package notation, `gcluster create` will copy the entire -repository to the module path: `deployment_name/group_name/module_id`. However, -when `gcluster deploy` is invoked, it will run Packer from the subdirectory -`deployment_name/group_name/module_id/subdirectory/after/double_slash`. - -If the module does not use `//` package notation, `gcluster create` will copy -only the final directory in the path to `deployment_name/group_name/module_id`. - -In all cases, `gcluster create` will remove the `.git` directory from the packer -module to ensure that you can manage the entire deployment directory with its -own git versioning. - -##### GitHub over SSH - -Get module from GitHub over SSH: - -```yaml - - id: network1 - source: git@github.com:GoogleCloudPlatform/hpc-toolkit.git//modules/network/vpc -``` - -Specific versions can be selected as for HTTPS: - -```yaml - - id: network1 - source: git@github.com:GoogleCloudPlatform/hpc-toolkit.git//modules/network/vpc?ref=v1.22.1&depth=1 -``` - -##### Generic Git Modules - -To use a Terraform module available in a non-GitHub git repository such as -gitlab, set the source to a path starting `git::`. Two Standard git protocols -are supported, `git::https://` for HTTPS or `git::git@github.com` for SSH. - -Additional formatting and features after `git::` are identical to that of the -[GitHub Modules](#github-modules) described above. - -#### Google Cloud Storage Modules - -To use a Terraform module available in a Google Cloud Storage bucket, set the source -to a URL with the special `gcs::` prefix, followed by a [GCS bucket object URL](https://cloud.google.com/storage/docs/request-endpoints#typical). - -For example: `gcs::https://www.googleapis.com/storage/v1/BUCKET_NAME/PATH_TO_MODULE` - -### Kind (May be Required) - -`kind` refers to the way in which a module is deployed. Currently, `kind` can be -either `terraform` or `packer`. It must be specified for modules of type -`packer`. If omitted, it will default to `terraform`. - -### Settings (May Be Required) - -The settings field is a map that supplies any user-defined variables for each -module. Settings values can be simple strings, numbers or booleans, but can -also support complex data types like maps and lists of variable depth. These -settings will become the values for the variables defined in either the -`variables.tf` file for Terraform or `variable.pkr.hcl` file for Packer. - -For some modules, there are mandatory variables that must be set, -therefore `settings` is a required field in that case. In many situations, a -combination of sensible defaults, deployment variables and used modules can -populated all required settings and therefore the settings field can be omitted. - -### Use (Optional) - -The `use` field is a powerful way of linking a module to one or more other -modules. When a module "uses" another module, the outputs of the used -module are compared to the settings of the current module. If they have -matching names and the setting has no explicit value, then it will be set to -the used module's output. For example, see the following blueprint snippet: - -```yaml -modules: -- id: network1 - source: modules/network/vpc - -- id: workstation - source: modules/compute/vm-instance - use: [network1] - settings: - ... -``` - -In this snippet, the VM instance `workstation` uses the outputs of vpc -`network1`. - -In this case both `network_self_link` and `subnetwork_self_link` in the -[workstation settings](compute/vm-instance/README.md#Inputs) will be set -to `$(network1.network_self_link)` and `$(network1.subnetwork_self_link)` which -refer to the [network1 outputs](network/vpc/README#Outputs) -of the same names. - -The order of precedence that `gcluster` uses in determining when to infer a setting -value is in the following priority order: - -1. Explicitly set in the blueprint using the `settings` field -1. Output from a used module, taken in the order provided in the `use` list -1. Deployment variable (`vars`) of the same name -1. Default value for the setting - -> **_NOTE:_** See the -> [network storage documentation](./../docs/network_storage.md) for more -> information about mounting network storage file systems via the `use` field. - -### Outputs (Optional) - -The `outputs` field adds the output of individual Terraform modules to the -output of its deployment group. This enables the value to be available via -`terraform output`. This can useful for displaying the IP of a login node or -printing instructions on how to use a module, as we have in the -[monitoring dashboard module](monitoring/dashboard/README.md#Outputs). - -The outputs field is a lists that it can be in either of two formats: a string -equal to the name of the module output, or a map specifying the `name`, -`description`, and whether the value is `sensitive` and should be suppressed -from the standard output of Terraform commands. An example is shown below -that displays the internal and public IP addresses of a VM created by the -vm-instance module: - -```yaml - - id: vm - source: modules/compute/vm-instance - use: - - network1 - settings: - machine_type: e2-medium - outputs: - - internal_ip - - name: external_ip - description: "External IP of VM" - sensitive: true -``` - -The outputs shown after running Terraform apply will resemble: - -```text -Apply complete! Resources: 7 added, 0 changed, 0 destroyed. - -Outputs: - -external_ip_simplevm = -internal_ip_simplevm = [ - "10.128.0.19", -] -``` - -### Required Services (APIs) (optional) - -Each Toolkit module depends upon Google Cloud services ("APIs") being enabled -in the project used by the AI/ML and HPC environment. For example, the [creation of -VMs](compute/vm-instance/) requires the Compute Engine API -(compute.googleapis.com). The [startup-script](scripts/startup-script/) module -requires the Cloud Storage API (storage.googleapis.com) for storage of the -scripts themselves. Each module included in the Toolkit source code describes -its required APIs internally. The Toolkit will merge the requirements from all -modules and [automatically validate](../README.md#blueprint-validation) that all -APIs are enabled in the project specified by `$(vars.project_id)`. - -## Common Settings - -The following common naming conventions should be used to decrease the verbosity -needed to define a blueprint. This is intentional to allow multiple -modules to share inferred settings from deployment variables or from other -modules listed under the `use` field. - -For example, if all modules are to be created in a single region, that region -can be defined as a deployment variable named `region`, which is shared between -all modules without an explicit setting. Similarly, if many modules need to be -connected to the same VPC network, they all can add the vpc module ID to their -`use` list so that `network_self_link` would be inferred from that vpc module rather -than having to set it manually. - -* **project_id**: The GCP project ID in which to create the GCP resources. -* **deployment_name**: The name of the current deployment of a blueprint. This - can help to avoid naming conflicts of modules when multiple deployments are - created from the same blueprint. -* **region**: The GCP - [region](https://cloud.google.com/compute/docs/regions-zones) the module - will be created in. -* **zone**: The GCP [zone](https://cloud.google.com/compute/docs/regions-zones) - the module will be created in. -* **labels**: - [Labels](https://cloud.google.com/resource-manager/docs/creating-managing-labels) - added to the module. In order to include any module in advanced - monitoring, labels must be exposed. We strongly recommend that all modules - expose this variable. - -## Writing Custom Cluster Toolkit Modules - -Modules are flexible by design, however we define some [best practices](../docs/module-guidelines.md) when -creating a new module meant to be used with the Cluster Toolkit. diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/README.md b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/README.md deleted file mode 100644 index 7c49141d1f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/README.md +++ /dev/null @@ -1,130 +0,0 @@ -## Description - -This module is used to create a Kubernetes job template file. - -The job template file can be submitted as is or used as a template for further -customization. Add the `instructions` output to a blueprint (as shown below) to -get instructions on how to use `kubectl` to submit the job. - -This module is designed to `use` one or more `gke-node-pool` modules. The job -will be configured to run on any of the specified node pools. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -### Example - -The following example creates a GKE job template file. - -```yaml - - id: job-template - source: modules/compute/gke-job-template - use: [compute_pool] - settings: - node_count: 3 - outputs: [instructions] -``` - -Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). - -### Storage Options - -This module natively supports: - -* Filestore as a shared file system between pods/nodes. -* Pod level ephemeral storage options: - * memory backed emptyDir - * local SSD backed emptyDir - * SSD persistent disk backed ephemeral volume - * balanced persistent disk backed ephemeral volume - -See the [storage-gke.yaml blueprint](../../../examples/storage-gke.yaml) and the -associated [documentation](../../../../examples/README.md#storage-gkeyaml--) for -examples of how to use Filestore and ephemeral storage with this module. - -### Requested Resources - -When one or more `gke-node-pool` modules are referenced with the `use` field. -The requested resources will be populated to achieve a 1 pod per node packing -while still leaving some headroom for required system pods. - -This functionality can be overridden by specifying the desired cpu requirement -using the `requested_cpu_per_pod` setting. - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.2 | -| [local](#requirement\_local) | >= 2.0.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [local](#provider\_local) | >= 2.0.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [local_file.job_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allocatable\_cpu\_per\_node](#input\_allocatable\_cpu\_per\_node) | The allocatable cpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field. | `list(number)` | 
[
  -1
]
| no | -| [allocatable\_gpu\_per\_node](#input\_allocatable\_gpu\_per\_node) | The allocatable gpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field. | `list(number)` | 
[
  -1
]
| no | -| [backoff\_limit](#input\_backoff\_limit) | Controls the number of retries before considering a Job as failed. Set to zero for shared fate. | `number` | `0` | no | -| [command](#input\_command) | The command and arguments for the container that run in the Pod. The command field corresponds to entrypoint in some container runtimes. | `list(string)` | 
[
  "hostname"
]
| no | -| [completion\_mode](#input\_completion\_mode) | Sets value of `completionMode` on the job. Default uses indexed jobs. See [documentation](https://kubernetes.io/blog/2021/04/19/introducing-indexed-jobs/) for more information | `string` | `"Indexed"` | no | -| [ephemeral\_volumes](#input\_ephemeral\_volumes) | Will create an emptyDir or ephemeral volume that is backed by the specified type: `memory`, `local-ssd`, `pd-balanced`, `pd-ssd`. `size_gb` is provided in GiB. | 
list(object({
    type       = string
    mount_path = string
    size_gb    = number
  }))
| `[]` | no | -| [has\_gpu](#input\_has\_gpu) | Indicates that the job should request nodes with GPUs. Typically supplied by a gke-node-pool module. | `list(bool)` | 
[
  false
]
| no | -| [image](#input\_image) | The container image the job should use. | `string` | `"debian"` | no | -| [k8s\_service\_account\_name](#input\_k8s\_service\_account\_name) | Kubernetes service account to run the job as. If null then no service account is specified. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to the GKE job template. Key-value pairs. | `map(string)` | n/a | yes | -| [machine\_family](#input\_machine\_family) | The machine family to use in the node selector (example: `n2`). If null then machine family will not be used as selector criteria. | `string` | `null` | no | -| [name](#input\_name) | The name of the job. | `string` | `"my-job"` | no | -| [node\_count](#input\_node\_count) | How many nodes the job should run in parallel. | `number` | `1` | no | -| [node\_pool\_names](#input\_node\_pool\_names) | A list of node pool names on which to run the job. Can be populated via `use` field. | `list(string)` | `[]` | no | -| [node\_selectors](#input\_node\_selectors) | A list of node selectors to use to place the job. | 
list(object({
    key   = string
    value = string
  }))
| `[]` | no | -| [persistent\_volume\_claims](#input\_persistent\_volume\_claims) | A list of objects that describes a k8s PVC that is to be used and mounted on the job. Generally supplied by the gke-persistent-volume module. | 
list(object({
    name          = string
    mount_path    = string
    mount_options = string
    is_gcs        = bool
  }))
| `[]` | no | -| [random\_name\_sufix](#input\_random\_name\_sufix) | Appends a random suffix to the job name to avoid clashes. | `bool` | `true` | no | -| [requested\_cpu\_per\_pod](#input\_requested\_cpu\_per\_pod) | The requested cpu per pod. If null, allocatable\_cpu\_per\_node will be used to claim whole nodes. If provided will override allocatable\_cpu\_per\_node. | `number` | `-1` | no | -| [requested\_gpu\_per\_pod](#input\_requested\_gpu\_per\_pod) | The requested gpu per pod. If null, allocatable\_gpu\_per\_node will be used to claim whole nodes. If provided will override allocatable\_gpu\_per\_node. | `number` | `-1` | no | -| [restart\_policy](#input\_restart\_policy) | Job restart policy. Only a RestartPolicy equal to `Never` or `OnFailure` is allowed. | `string` | `"Never"` | no | -| [security\_context](#input\_security\_context) | The security options the container should be run with. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ | 
list(object({
    key   = string
    value = string
  }))
| `[]` | no | -| [tolerations](#input\_tolerations) | Tolerations allow the scheduler to schedule pods with matching taints. Generally populated from gke-node-pool via `use` field. | 
list(object({
    key      = string
    operator = string
    value    = string
    effect   = string
  }))
| 
[
  {
    "effect": "NoSchedule",
    "key": "user-workload",
    "operator": "Equal",
    "value": "true"
  }
]
| no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instructions](#output\_instructions) | Instructions for submitting the GKE job. | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/main.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/main.tf deleted file mode 100644 index 835e3ca874..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/main.tf +++ /dev/null @@ -1,168 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "gke-job-template", ghpc_role = "compute" }) -} - -locals { - # Start with the minimum cpu available of used node pools - min_allocatable_cpu = min(var.allocatable_cpu_per_node...) - full_node_cpu_request = ( - local.min_allocatable_cpu > 2 ? # if large enough - local.min_allocatable_cpu - 1 : # leave headroom for 1 cpu - local.min_allocatable_cpu / 2 + 0.1 # else take just over half - ) - (local.any_gcs ? 0.25 : 0) # save room for gcs side car - - cpu_request = ( - var.requested_cpu_per_pod >= 0 ? # if user supplied requested cpu - var.requested_cpu_per_pod : # then honor it - ( # else - local.min_allocatable_cpu >= 0 ? # if allocatable cpu was supplied - local.full_node_cpu_request : # then claim the full node - -1 # else do not set a limit - ) - ) - millicpu = floor(local.cpu_request * 1000) - cpu_request_string = local.millicpu >= 0 ? "${local.millicpu}m" : null - full_node_request = local.min_allocatable_cpu >= 0 && var.requested_cpu_per_pod < 0 - - memory_request_value = try(sum([for ed in var.ephemeral_volumes : - ed.size_gb - if ed.type == "memory" - ]), 0) - memory_request_string = local.memory_request_value > 0 ? "${local.memory_request_value}Gi" : null - - ephemeral_request_value = try(sum([for ed in var.ephemeral_volumes : - ed.size_gb - if ed.type == "local-ssd" - ]), 0) - ephemeral_request_string = local.ephemeral_request_value > 0 ? "${local.ephemeral_request_value}Gi" : null - - uses_local_ssd = anytrue([for ed in var.ephemeral_volumes : - ed.type == "local-ssd" - ]) - local_ssd_node_selector = local.uses_local_ssd ? [{ - key = "cloud.google.com/gke-ephemeral-storage-local-ssd" - value = "true" - }] : [] - - # Setup limit for GPUs per pod - min_allocatable_gpu = min(var.allocatable_gpu_per_node...) - min_allocatable_gpu_per_pod = local.min_allocatable_gpu > 0 ? local.min_allocatable_gpu : null - gpu_limit_per_pod = var.requested_gpu_per_pod > 0 ? var.requested_gpu_per_pod : local.min_allocatable_gpu_per_pod - gpu_limit_string = alltrue(var.has_gpu) ? tostring(local.gpu_limit_per_pod) : null - - empty_dir_volumes = [for ed in var.ephemeral_volumes : - { - name = replace(trim(ed.mount_path, "/"), "/", "-") - mount_path = ed.mount_path - size_limit = "${ed.size_gb}Gi" - in_memory = ed.type == "memory" - } - if contains(["memory", "local-ssd"], ed.type) - ] - - ephemeral_pd_volumes = [for pd in var.ephemeral_volumes : - { - name = replace(trim(pd.mount_path, "/"), "/", "-") - mount_path = pd.mount_path - storage_class_name = pd.type == "pd-ssd" ? "premium-rwo" : "standard-rwo" - storage = "${pd.size_gb}Gi" - } - if contains(["pd-balanced", "pd-ssd"], pd.type) - ] - - pvc_volumes = [for pvc in var.persistent_volume_claims : - { - name = replace(trim(pvc.mount_path, "/"), "/", "-") - mount_path = pvc.mount_path - claim_name = pvc.name - } - ] - - volume_mounts = [for v in concat(local.empty_dir_volumes, local.ephemeral_pd_volumes, local.pvc_volumes) : - { - name = v.name - mount_path = v.mount_path - } - ] - - suffix = var.random_name_sufix ? "-${random_id.resource_name_suffix.hex}" : "" - machine_family_node_selector = var.machine_family != null ? [{ - key = "cloud.google.com/machine-family" - value = var.machine_family - }] : [] - node_selectors = concat(local.machine_family_node_selector, local.local_ssd_node_selector, var.node_selectors) - - any_gcs = anytrue([for pvc in var.persistent_volume_claims : - pvc.is_gcs - ]) - - job_template_contents = templatefile( - "${path.module}/templates/gke-job-base.yaml.tftpl", - { - name = var.name - suffix = local.suffix - image = var.image - command = var.command - node_count = var.node_count - completion_mode = var.completion_mode - k8s_service_account_name = var.k8s_service_account_name - node_pool_names = var.node_pool_names - node_selectors = local.node_selectors - full_node_request = local.full_node_request - cpu_request = local.cpu_request_string - gpu_limit = local.gpu_limit_string - restart_policy = var.restart_policy - backoff_limit = var.backoff_limit - tolerations = distinct(var.tolerations) - security_context = var.security_context - labels = local.labels - - empty_dir_volumes = local.empty_dir_volumes - ephemeral_pd_volumes = local.ephemeral_pd_volumes - pvc_volumes = local.pvc_volumes - volume_mounts = local.volume_mounts - memory_request = local.memory_request_string - ephemeral_request = local.ephemeral_request_string - gcs_annotation = local.any_gcs - } - ) - - job_template_output_path = "${path.root}/${var.name}${local.suffix}.yaml" - -} - -resource "random_id" "resource_name_suffix" { - byte_length = 2 - keepers = { - timestamp = timestamp() - } -} - -resource "local_file" "job_template" { - content = local.job_template_contents - filename = local.job_template_output_path - - lifecycle { - precondition { - condition = local.any_gcs ? var.k8s_service_account_name != null : true - error_message = "When using GCS, a kubernetes service account with workload identity is required. gke-cluster module will perform this setup when var.configure_workload_identity_sa is set to true." - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/outputs.tf deleted file mode 100644 index adf78e936d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/outputs.tf +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "instructions" { - description = "Instructions for submitting the GKE job." - value = <<-EOT - A GKE job file has been created locally at: - ${abspath(local.job_template_output_path)} - - Use the following commands to: - Submit your job: - kubectl create -f ${abspath(local.job_template_output_path)} - EOT -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl deleted file mode 100644 index 431a519b9c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/templates/gke-job-base.yaml.tftpl +++ /dev/null @@ -1,120 +0,0 @@ ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: ${name}${suffix} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - parallelism: ${node_count} - completions: ${node_count} - completionMode: ${completion_mode} - template: - %{~ if gcs_annotation ~} - metadata: - annotations: - gke-gcsfuse/volumes: "true" - %{~ endif ~} - spec: - %{~ if length(security_context) > 0 ~} - securityContext: - %{~ for context in security_context ~} - ${context.key}: ${context.value} - %{~ endfor ~} - %{~ endif ~} - %{~ if k8s_service_account_name != null ~} - serviceAccountName: ${k8s_service_account_name} - %{~ endif ~} - %{~ if length(node_pool_names) > 0 ~} - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: cloud.google.com/gke-nodepool - operator: In - values: - %{~ for node_pool in node_pool_names ~} - - ${node_pool} - %{~ endfor ~} - %{~ endif ~} - %{~ if length(node_selectors) > 0 ~} - nodeSelector: - %{~ for selector in node_selectors ~} - ${selector.key}: "${selector.value}" - %{~ endfor ~} - %{~ endif ~} - tolerations: - %{~ for toleration in tolerations ~} - - key: ${toleration.key} - operator: ${toleration.operator} - value: "${toleration.value}" - effect: ${toleration.effect} - %{~ endfor ~} - containers: - - name: ${name}-container - image: ${image} - command: - %{for s in command}- ${indent(8, yamlencode(s))}%{~ endfor } - %{~ if gpu_limit != null || cpu_request != null ~} - resources: - %{~ if gpu_limit != null ~} - limits: - # GPUs should only be specified as limits - # https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/ - nvidia.com/gpu: ${gpu_limit} - %{~ endif ~} - %{~ if cpu_request != null || memory_request != null || ephemeral_request != null ~} - requests: - %{~ if full_node_request ~} - # cpu request attempts full node per pod - %{~ endif ~} - %{~ if cpu_request != null ~} - cpu: ${cpu_request} - %{~ endif ~} - %{~ if memory_request != null ~} - memory: ${memory_request} - %{~ endif ~} - %{~ if ephemeral_request != null ~} - ephemeral-storage: ${ephemeral_request} - %{~ endif ~} - %{~ endif ~} - %{~ endif ~} - %{~ if length(volume_mounts) > 0 ~} - volumeMounts: - %{~ for v in volume_mounts ~} - - name: ${v.name} - mountPath: ${v.mount_path} - %{~ endfor ~} - %{~ endif ~} - %{~ if length(volume_mounts) > 0 ~} - volumes: - %{~ for ed in empty_dir_volumes ~} - - name: ${ed.name} - emptyDir: - sizeLimit: ${ed.size_limit} - %{~ if ed.in_memory ~} - medium: "Memory" - %{~ endif ~} - %{~ endfor ~} - %{~ for pd in ephemeral_pd_volumes ~} - - name: ${pd.name} - ephemeral: - volumeClaimTemplate: - spec: - accessModes: [ "ReadWriteOnce" ] - storageClassName: ${pd.storage_class_name} - resources: - requests: - storage: ${pd.storage} - %{~ endfor ~} - %{~ for pvc in pvc_volumes ~} - - name: ${pvc.name} - persistentVolumeClaim: - claimName: ${pvc.claim_name} - %{~ endfor ~} - %{~ endif ~} - restartPolicy: ${restart_policy} - backoffLimit: ${backoff_limit} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/variables.tf deleted file mode 100644 index f849831bf4..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/variables.tf +++ /dev/null @@ -1,187 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "name" { - description = "The name of the job." - type = string - default = "my-job" -} - -variable "node_count" { - description = "How many nodes the job should run in parallel." - type = number - default = 1 -} - -variable "completion_mode" { - description = "Sets value of `completionMode` on the job. Default uses indexed jobs. See [documentation](https://kubernetes.io/blog/2021/04/19/introducing-indexed-jobs/) for more information" - type = string - default = "Indexed" -} - -variable "command" { - description = "The command and arguments for the container that run in the Pod. The command field corresponds to entrypoint in some container runtimes." - type = list(string) - default = ["hostname"] -} - -variable "image" { - description = "The container image the job should use." - type = string - default = "debian" -} - -variable "k8s_service_account_name" { - description = "Kubernetes service account to run the job as. If null then no service account is specified." - type = string - default = null -} - -variable "node_pool_names" { - description = "A list of node pool names on which to run the job. Can be populated via `use` field." - type = list(string) - default = [] -} - -variable "allocatable_cpu_per_node" { - description = "The allocatable cpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field." - type = list(number) - default = [-1] -} - -variable "has_gpu" { - description = "Indicates that the job should request nodes with GPUs. Typically supplied by a gke-node-pool module." - type = list(bool) - default = [false] -} - -variable "requested_cpu_per_pod" { - description = "The requested cpu per pod. If null, allocatable_cpu_per_node will be used to claim whole nodes. If provided will override allocatable_cpu_per_node." - type = number - default = -1 -} - -variable "allocatable_gpu_per_node" { - description = "The allocatable gpu per node. Used to claim whole nodes. Generally populated from gke-node-pool via `use` field." - type = list(number) - default = [-1] -} - -variable "requested_gpu_per_pod" { - description = "The requested gpu per pod. If null, allocatable_gpu_per_node will be used to claim whole nodes. If provided will override allocatable_gpu_per_node." - type = number - default = -1 -} - -variable "tolerations" { - description = "Tolerations allow the scheduler to schedule pods with matching taints. Generally populated from gke-node-pool via `use` field." - type = list(object({ - key = string - operator = string - value = string - effect = string - })) - default = [ - { - key = "user-workload" - operator = "Equal" - value = "true" - effect = "NoSchedule" - } - ] -} - -variable "security_context" { - description = "The security options the container should be run with. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/" - type = list(object({ - key = string - value = string - })) - default = [] -} - -variable "machine_family" { - description = "The machine family to use in the node selector (example: `n2`). If null then machine family will not be used as selector criteria." - type = string - default = null -} - -variable "node_selectors" { - description = "A list of node selectors to use to place the job." - type = list(object({ - key = string - value = string - })) - default = [] -} - -variable "restart_policy" { - description = "Job restart policy. Only a RestartPolicy equal to `Never` or `OnFailure` is allowed." - type = string - default = "Never" -} - -variable "backoff_limit" { - description = "Controls the number of retries before considering a Job as failed. Set to zero for shared fate." - type = number - default = 0 -} - -variable "random_name_sufix" { - description = "Appends a random suffix to the job name to avoid clashes." - type = bool - default = true -} - -variable "persistent_volume_claims" { - description = "A list of objects that describes a k8s PVC that is to be used and mounted on the job. Generally supplied by the gke-persistent-volume module." - type = list(object({ - name = string - mount_path = string - mount_options = string - is_gcs = bool - })) - default = [] -} - -variable "ephemeral_volumes" { - description = "Will create an emptyDir or ephemeral volume that is backed by the specified type: `memory`, `local-ssd`, `pd-balanced`, `pd-ssd`. `size_gb` is provided in GiB." - type = list(object({ - type = string - mount_path = string - size_gb = number - })) - default = [] - validation { - condition = alltrue([ - for v in var.ephemeral_volumes : - contains(["pd-balanced", "pd-ssd", "memory", "local-ssd"], v.type) - ]) - error_message = "Type must be one of 'pd-balanced', 'pd-ssd', 'memory', 'local-ssd'." - } - validation { - condition = alltrue([ - for v in var.ephemeral_volumes : - substr(v.mount_path, 0, 1) == "/" - ]) - error_message = "Mount path must start with the '/' character." - } -} - -variable "labels" { - description = "Labels to add to the GKE job template. Key-value pairs." - type = map(string) -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/versions.tf deleted file mode 100644 index 0f902ac8c5..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-job-template/versions.tf +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.2" - - required_providers { - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/README.md b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/README.md deleted file mode 100644 index c8b0591544..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/README.md +++ /dev/null @@ -1,375 +0,0 @@ -## Description - -This module creates a Google Kubernetes Engine -([GKE](https://cloud.google.com/kubernetes-engine)) node pool. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -### Example - -The following example creates a GKE node group. - -```yaml - - id: compute_pool - source: modules/compute/gke-node-pool - use: [gke_cluster] -``` - -Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). - -### Taints and Tolerations - -By default node pools created with this module will be tainted with -`user-workload=true:NoSchedule` to prevent system pods from being scheduled. -User jobs targeting the node pool should include this toleration. This behavior -can be overridden using the `taints` setting. See -[docs](https://cloud.google.com/kubernetes-engine/docs/how-to/node-taints) for -more info. - -### Local SSD Storage -GKE offers two options for managing locally attached SSDs. - -The first, and recommended, option is for GKE to manage the ephemeral storage -space on the node, which will then be automatically attached to pods which -request an `emptyDir` volume. This can be accomplished using the -[`local_ssd_count_ephemeral_storage`] variable. - -The second, more complex, option is for GCP to attach these nodes as raw block -storage. In this case, the cluster administrator is responsible for software -RAID settings, partitioning, formatting and mounting these disks on the host -OS. Still, this may be desired behavior in use cases which aren't supported -by an `emptyDir` volume (for example, a `ReadOnlyMany` or `ReadWriteMany` PV). -This can be accomplished using the [`local_ssd_count_nvme_block`] variable. - -The [`local_ssd_count_ephemeral_storage`] and [`local_ssd_count_nvme_block`] -variables are mutually exclusive and cannot be mixed together. - -Also, the number of SSDs which can be attached to a node depends on the -[machine type](https://cloud.google.com/compute/docs/disks#local_ssd_machine_type_restrictions). - -See [docs](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/local-ssd) -for more info. - -[`local_ssd_count_ephemeral_storage`]: #input\_local\_ssd\_count\_ephemeral\_storage -[`local_ssd_count_nvme_block`]: #input\_local\_ssd\_count\_nvme\_block - -### Considerations with GPUs - -When a GPU is attached to a node an additional taint is automatically added: -`nvidia.com/gpu=present:NoSchedule`. For jobs to get placed on these nodes, the -equivalent toleration is required. The `gke-job-template` module will -automatically apply this toleration when using a node pool with GPUs. - -Nvidia GPU drivers must be installed. The recommended approach for GKE to install -GPU dirvers is by applying a DaemonSet to the cluster. See -[these instructions](https://cloud.google.com/kubernetes-engine/docs/how-to/gpus#cos). - -However, in some cases it may be desired to compile a different driver (such as -a desire to install a newer version, compatibility with the -[Nvidia GPU-operator](https://github.com/NVIDIA/gpu-operator) or other -use-cases). In this case, ensure that you turn off the -[enable_secure_boot](#input\_enable\_secure\_boot) option to allow unsigned -kernel modules to be loaded. - -#### Maximize GPU network bandwidth with GPUDirect and multi-networking -For A3 Series machines to achieve optimal performance , GKE provide two networking stacks for remote direct memory access (RDMA): - -- A3 High machine types (a3-highgpu-8g): utilize GPUDirect-TCPX to reduce the overhead required to transfer packet payloads to and from GPUs, which significantly improves throughput at scale compared to GPUs that don't use GPUDirect. -- A3 Mega machine types (a3-megagpu-8g): utilize GPUDirect-TCPXO to improve GPU to GPU communication, and further improves GPU to VM communication. - -To achieve this, when creating nodepools with A3 Series machine type, pass in a multivpc module to the gke-node-pool module, and the gke-node-pool module would detect the eligible machine type and enable GPUDirect for it. More specifically, the below components will be installed in the nodepool for enabling GPUDirect. - -- Install NCCL plugin for GPUDirect [TCPX](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/gpudirect-tcpx) or [TCPXO](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/gpudirect-tcpxo) -- Install [NRI](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/nri_device_injector) device injector plugin -- Provide support for injecting GPUDirect required components(annotations, volumes, rxdm sidecar etc.) into the user workload in the form of Kubernetes Job. - - Provide sample workload to showcase how it will be updated with the required components injected, and how it can be deployed. - - Allow user to use the provided script to update their own workload and deploy. - -The GPUDirect supports included in the Cluster Toolkit aim to automate the [GPUDirect User Guid](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#install-gpudirect-tcpx-nccl) and provide better usability. - -> **_NOTE:_** You must [enable multi networking](https://cloud.google.com/kubernetes-engine/docs/how-to/setup-multinetwork-support-for-pods#create-a-gke-cluster) feature when creating the GKE cluster. When gke-cluster depends on multivpc (with the use keyword), multi networking will be automatically enabled on the cluster creation. -> When gke-cluster or pre-existing-gke-cluster depends on multivpc (with the use keyword), the [network objects](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment) required for multi networking will be created on the cluster. - -### GPUs Examples - -There are several ways to add GPUs to a GKE node pool. See -[docs](https://cloud.google.com/compute/docs/gpus) for more info on GPUs. - -The following is a node pool that uses `a2`, `a3` or `g2` machine types which has a -fixed number of attached GPUs, let's call these machine types as "pre-defined gpu machine families": - -```yaml - - id: simple-a2-pool - source: modules/compute/gke-node-pool - use: [gke_cluster] - settings: - machine_type: a2-highgpu-1g -``` - -> **Note**: It is not necessary to define the [`guest_accelerator`] setting when -> using pre-defined gpu machine families as information about GPUs, such as type, count and -> `gpu_driver_installation_config`, is automatically inferred from the machine type. -> Optional fields such as `gpu_partition_size` need to be specified only if they have -> non-default values. - -The following scenarios require the [`guest_accelerator`] block is specified: - -- To partition an A100 GPU into multiple GPUs on an A2 family machine. -- To specify a time sharing configuration on a GPUs. -- To attach a GPU to an N1 family machine. - -The following is an example of -[partitioning](https://cloud.google.com/kubernetes-engine/docs/how-to/gpus-multi) -an A100 GPU: - -> **Note**: In the following example, `type`, `count` and `gpu_driver_installation_config` are picked up automatically. - -```yaml - - id: multi-instance-gpu-pool - source: modules/compute/gke-node-pool - use: [gke_cluster] - settings: - machine_type: a2-highgpu-1g - guest_accelerator: - - gpu_partition_size: 1g.5gb -``` - -[`guest_accelerator`]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_guest_accelerator - -The following is an example of -[GPU time sharing](https://cloud.google.com/kubernetes-engine/docs/concepts/timesharing-gpus) -(with partitioned GPUs): - -```yaml - - id: time-sharing-gpu-pool - source: modules/compute/gke-node-pool - use: [gke_cluster] - settings: - machine_type: a2-highgpu-1g - guest_accelerator: - - gpu_partition_size: 1g.5gb - gpu_sharing_config: - gpu_sharing_strategy: TIME_SHARING - max_shared_clients_per_gpu: 3 -``` - -Following is an example of using a GPU attached to an `n1` machine: - -```yaml - - id: t4-pool - source: modules/compute/gke-node-pool - use: [gke_cluster] - settings: - machine_type: n1-standard-16 - guest_accelerator: - - type: nvidia-tesla-t4 - count: 2 -``` - -The following is an example of using a GPU (with sharing config) attached to an `n1` machine: - -```yaml - - id: n1-t4-pool - source: community/modules/compute/gke-node-pool - use: [gke_cluster] - settings: - name: n1-t4-pool - machine_type: n1-standard-1 - guest_accelerator: - - type: nvidia-tesla-t4 - count: 2 - gpu_driver_installation_config: - gpu_driver_version: "LATEST" - gpu_sharing_config: - max_shared_clients_per_gpu: 2 - gpu_sharing_strategy: "TIME_SHARING" -``` - -Finally, the following is adding multivpc to a node pool: - -```yaml - - id: network - source: modules/network/vpc - settings: - subnetwork_name: gke-subnet - secondary_ranges: - gke-subnet: - - range_name: pods - ip_cidr_range: 10.4.0.0/14 - - range_name: services - ip_cidr_range: 10.0.32.0/20 - - - id: multinetwork - source: modules/network/multivpc - settings: - network_name_prefix: multivpc-net - network_count: 8 - global_ip_address_range: 172.16.0.0/12 - subnetwork_cidr_suffix: 16 - - - id: gke-cluster - source: modules/scheduler/gke-cluster - use: [network, multinetwork] - settings: - cluster_name: $(vars.deployment_name) - - - id: a3-megagpu_pool - source: modules/compute/gke-node-pool - use: [gke-cluster, multinetwork] - settings: - machine_type: a3-megagpu-8g - ... -``` - -## Using GCE Reservations -You can reserve Google Compute Engine instances in a specific zone to ensure resources are available for their workloads when needed. For more details on how to manage reservations, see [Reserving Compute Engine zonal resources](https://cloud.google.com/compute/docs/instances/reserving-zonal-resources). - -After creating a reservation, you can consume the reserved GCE VM instances in GKE. GKE clusters deployed using Cluster Toolkit support the same consumption modes as Compute Engine: NO_RESERVATION(default), ANY_RESERVATION, SPECIFIC_RESERVATION. - -This can be accomplished using [`reservation_affinity`](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/compute/gke-node-pool/README.md#input_reservation_affinity). - -```yaml -# Target any reservation -reservation_affinity: - consume_reservation_type: ANY_RESERVATION - -# Target a specific reservation -reservation_affinity: - consume_reservation_type: SPECIFIC_RESERVATION - specific_reservations: - - name: specific-reservation-1 -``` - -The following requirements need to be satisfied for the node pool nodes to be able to use a specific reservation: -1. A reservation with the name must exist in the specified project(`var.project_id`) and one of the specified zones(`var.zones`). -2. Its consumption type must be `specific`. -3. Its GCE VM Properties must match with those of the Node Pool; Machine type, Accelerators (GPU Type and count), Local SSD disk type and count. - -If you want to utilise a shared reservation, the owner project of the shared reservation needs to be explicitly specified like the following. Note that a shared reservation can be used by the project that hosts the reservation (owner project) and by the projects the reservation is shared with (consumer projects). See how to [create and use a shared reservation](https://cloud.google.com/compute/docs/instances/reservations-shared). - -```yaml -reservation_affinity: - consume_reservation_type: SPECIFIC_RESERVATION - specific_reservations: - - name: specific-reservation-shared - project: shared_reservation_owner_project_id -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [google](#requirement\_google) | >= 6.41 | -| [null](#requirement\_null) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.41 | -| [null](#provider\_null) | ~> 3.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [gpu](#module\_gpu) | ../../internal/gpu-definition | n/a | -| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_container_node_pool.node_pool](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool) | resource | -| [null_resource.enable_tcpx_in_workload](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.enable_tcpxo_in_workload](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.install_dependencies](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_machine_types.machine_info](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_machine_types) | data source | -| [google_compute_reservation.specific_reservations](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_reservation) | data source | -| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks adds additional node networks to the node pool | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [auto\_repair](#input\_auto\_repair) | Whether the nodes will be automatically repaired. | `bool` | `true` | no | -| [auto\_upgrade](#input\_auto\_upgrade) | Whether the nodes will be automatically upgraded. | `bool` | `false` | no | -| [autoscaling\_total\_max\_nodes](#input\_autoscaling\_total\_max\_nodes) | Total maximum number of nodes in the NodePool. | `number` | `1000` | no | -| [autoscaling\_total\_min\_nodes](#input\_autoscaling\_total\_min\_nodes) | Total minimum number of nodes in the NodePool. | `number` | `0` | no | -| [cluster\_id](#input\_cluster\_id) | projects/{{project}}/locations/{{location}}/clusters/{{cluster}} | `string` | n/a | yes | -| [compact\_placement](#input\_compact\_placement) | DEPRECATED: Use `placement_policy` | `bool` | `null` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for each node. | `number` | `100` | no | -| [disk\_type](#input\_disk\_type) | Disk type for each node. | `string` | `null` | no | -| [enable\_flex\_start](#input\_enable\_flex\_start) | If true, start the node pool with Flex Start provisioning model.
To learn more about flex-start mode, please refer to
https://cloud.google.com/kubernetes-engine/docs/how-to/dws-flex-start-training and
https://cloud.google.com/kubernetes-engine/docs/how-to/provisioningrequest | `bool` | `false` | no | -| [enable\_gcfs](#input\_enable\_gcfs) | Enable the Google Container Filesystem (GCFS). See [restrictions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#gcfs_config). | `bool` | `false` | no | -| [enable\_private\_nodes](#input\_enable\_private\_nodes) | Whether nodes have internal IP addresses only. | `bool` | `false` | no | -| [enable\_queued\_provisioning](#input\_enable\_queued\_provisioning) | If true, enables Dynamic Workload Scheduler and adds the cloud.google.com/gke-queued taint to the node pool. | `bool` | `false` | no | -| [enable\_secure\_boot](#input\_enable\_secure\_boot) | Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info. | `bool` | `true` | no | -| [gke\_version](#input\_gke\_version) | GKE version | `string` | n/a | yes | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = optional(string)
    count = optional(number, 0)
    gpu_driver_installation_config = optional(object({
      gpu_driver_version = string
    }), { gpu_driver_version = "DEFAULT" })
    gpu_partition_size = optional(string)
    gpu_sharing_config = optional(object({
      gpu_sharing_strategy       = string
      max_shared_clients_per_gpu = number
    }))
  }))
| `[]` | no | -| [host\_maintenance\_interval](#input\_host\_maintenance\_interval) | Specifies the frequency of planned maintenance events. | `string` | `""` | no | -| [image\_type](#input\_image\_type) | The default image type used by NAP once a new node pool is being created. Use either COS\_CONTAINERD or UBUNTU\_CONTAINERD. | `string` | `"COS_CONTAINERD"` | no | -| [initial\_node\_count](#input\_initial\_node\_count) | The initial number of nodes for the pool. In regional clusters, this is the number of nodes per zone. Changing this setting after node pool creation will not make any effect. It cannot be set with static\_node\_count and must be set to a value between autoscaling\_total\_min\_nodes and autoscaling\_total\_max\_nodes. | `number` | `null` | no | -| [internal\_ghpc\_module\_id](#input\_internal\_ghpc\_module\_id) | DO NOT SET THIS MANUALLY. Automatically populates with module id (unique blueprint-wide). | `string` | n/a | yes | -| [is\_reservation\_active](#input\_is\_reservation\_active) | Whether the specified reservation is already created. | `bool` | `true` | no | -| [kubernetes\_labels](#input\_kubernetes\_labels) | Kubernetes labels to be applied to each node in the node group. Key-value pairs.
(The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) | `map(string)` | `null` | no | -| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_ssd\_count\_ephemeral\_storage](#input\_local\_ssd\_count\_ephemeral\_storage) | The number of local SSDs to attach to each node to back ephemeral storage.
Uses NVMe interfaces. Must be supported by `machine_type`.
When set to null, default value either is [set based on machine\_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value.
[See above](#local-ssd-storage) for more info. | `number` | `null` | no | -| [local\_ssd\_count\_nvme\_block](#input\_local\_ssd\_count\_nvme\_block) | The number of local SSDs to attach to each node to back block storage.
Uses NVMe interfaces. Must be supported by `machine_type`.
When set to null, default value either is [set based on machine\_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value.
[See above](#local-ssd-storage) for more info. | `number` | `null` | no | -| [machine\_type](#input\_machine\_type) | The name of a Google Compute Engine machine type. | `string` | `"c2-standard-60"` | no | -| [max\_pods\_per\_node](#input\_max\_pods\_per\_node) | The maximum number of pods per node in this node pool. This will force replacement. | `number` | `null` | no | -| [max\_run\_duration](#input\_max\_run\_duration) | The duration (in whole seconds) of the instance. Instance will run and be terminated after then. | `number` | `null` | no | -| [name](#input\_name) | The name of the node pool. If not set, automatically populated by machine type and module id (unique blueprint-wide) as suffix.
If setting manually, ensure a unique value across all gke-node-pools. | `string` | `null` | no | -| [num\_node\_pools](#input\_num\_node\_pools) | Number of node pools to create. This is same as num\_slices. | `number` | `1` | no | -| [num\_slices](#input\_num\_slices) | Number of TPUs slices to create. This is same as num\_node\_pools. | `number` | `1` | no | -| [placement\_policy](#input\_placement\_policy) | Group placement policy to use for the node pool's nodes. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. `tpu_topology` is the TPU placement topology for pod slice node pool.
It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. | 
object({
    type         = string
    name         = optional(string)
    tpu_topology = optional(string)
  })
| 
{
  "name": null,
  "tpu_topology": null,
  "type": null
}
| no | -| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | -| [reservation\_affinity](#input\_reservation\_affinity) | Reservation resource to consume. When targeting SPECIFIC\_RESERVATION, specific\_reservations needs be specified.
Even though specific\_reservations is a list, only one reservation is allowed by the NodePool API.
It is assumed that the specified reservation exists and has available capacity.
For a shared reservation, specify the project\_id as well in which it was created.
To create a reservation refer to https://cloud.google.com/compute/docs/instances/reservations-single-project and https://cloud.google.com/compute/docs/instances/reservations-shared | 
object({
    consume_reservation_type = string
    specific_reservations = optional(list(object({
      name    = string
      project = optional(string)
    })))
  })
| 
{
  "consume_reservation_type": "NO_RESERVATION",
  "specific_reservations": []
}
| no | -| [run\_workload\_script](#input\_run\_workload\_script) | Whether execute the script to create a sample workload and inject rxdm sidecar into workload. Currently, implemented for A3-Highgpu and A3-Megagpu only. | `bool` | `true` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: use service\_account\_email and scopes. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the node pool | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [spot](#input\_spot) | Provision VMs using discounted Spot pricing, allowing for preemption | `bool` | `false` | no | -| [static\_node\_count](#input\_static\_node\_count) | The static number of nodes in the node pool. If set, autoscaling will be disabled. | `number` | `null` | no | -| [taints](#input\_taints) | Taints to be applied to the system node pool. | 
list(object({
    key    = string
    value  = any
    effect = string
  }))
| `[]` | no | -| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core. By setting threads\_per\_core
to 2, Simultaneous Multithreading (SMT) is enabled extending the total number
of virtual cores. For example, a machine of type c2-standard-60 will have 60
virtual cores with threads\_per\_core equal to 2. With threads\_per\_core equal
to 1 (SMT turned off), only the 30 physical cores will be available on the VM.

The default value of \"0\" will turn off SMT for supported machine types, and
will fall back to GCE defaults for unsupported machine types (t2d, shared-core
instances, or instances with less than 2 vCPU).

Disabling SMT can be more performant in many HPC workloads, therefore it is
disabled by default where compatible.

null = SMT configuration will use the GCE defaults for the machine type
0 = SMT will be disabled where compatible (default)
1 = SMT will always be disabled (will fail on incompatible machine types)
2 = SMT will always be enabled (will fail on incompatible machine types) | `number` | `0` | no | -| [timeout\_create](#input\_timeout\_create) | Timeout for creating a node pool | `string` | `null` | no | -| [timeout\_update](#input\_timeout\_update) | Timeout for updating a node pool | `string` | `null` | no | -| [total\_max\_nodes](#input\_total\_max\_nodes) | DEPRECATED: Use autoscaling\_total\_max\_nodes. | `number` | `null` | no | -| [total\_min\_nodes](#input\_total\_min\_nodes) | DEPRECATED: Use autoscaling\_total\_min\_nodes. | `number` | `null` | no | -| [upgrade\_settings](#input\_upgrade\_settings) | Defines node pool upgrade settings. It is highly recommended that you define all max\_surge and max\_unavailable.
If max\_surge is not specified, it would be set to a default value of 0.
If max\_unavailable is not specified, it would be set to a default value of 1. | 
object({
    strategy        = string
    max_surge       = optional(number)
    max_unavailable = optional(number)
  })
| 
{
  "max_surge": 0,
  "max_unavailable": 1,
  "strategy": "SURGE"
}
| no | -| [zones](#input\_zones) | A list of zones to be used. Zones must be in region of cluster. If null, cluster zones will be inherited. Note `zones` not `zone`; does not work with `zone` deployment variable. | `list(string)` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [allocatable\_cpu\_per\_node](#output\_allocatable\_cpu\_per\_node) | Number of CPUs available for scheduling pods on each node. | -| [allocatable\_gpu\_per\_node](#output\_allocatable\_gpu\_per\_node) | Number of GPUs available for scheduling pods on each node. | -| [has\_gpu](#output\_has\_gpu) | Boolean value indicating whether nodes in the pool are configured with GPUs. | -| [instructions](#output\_instructions) | Instructions for submitting the sample GPUDirect enabled job. | -| [node\_pool\_names](#output\_node\_pool\_names) | Names of the node pools. | -| [static\_gpu\_count](#output\_static\_gpu\_count) | Total number of GPUs in the node pool. Available only for static node pools. | -| [tolerations](#output\_tolerations) | Tolerations needed for a pod to be scheduled on this node pool. | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf deleted file mode 100644 index 0c1c255255..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/disk_definitions.tf +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -## Required variables: -# local_ssd_count_ephemeral_storage -# local_ssd_count_nvme_block -# machine_type - -locals { - - local_ssd_machines = { - "a3-highgpu-8g" = { local_ssd_count_ephemeral_storage = 16, local_ssd_count_nvme_block = null }, - "a3-megagpu-8g" = { local_ssd_count_ephemeral_storage = 16, local_ssd_count_nvme_block = null }, - "a3-ultragpu-8g" = { local_ssd_count_ephemeral_storage = 32, local_ssd_count_nvme_block = null }, - "a4-highgpu-8g" = { local_ssd_count_ephemeral_storage = 32, local_ssd_count_nvme_block = null }, - } - - generated_local_ssd_config = lookup(local.local_ssd_machines, var.machine_type, { local_ssd_count_ephemeral_storage = null, local_ssd_count_nvme_block = null }) - - # Select in priority order: - # (1) var.local_ssd_count_ephemeral_storage and var.local_ssd_count_nvme_block if any is not null - # (2) local.local_ssd_machines if not empty - # (3) default to null value for both local_ssd_count_ephemeral_storage and local_ssd_count_nvme_block - local_ssd_config = (var.local_ssd_count_ephemeral_storage == null && var.local_ssd_count_nvme_block == null) ? local.generated_local_ssd_config : { local_ssd_count_ephemeral_storage = var.local_ssd_count_ephemeral_storage, local_ssd_count_nvme_block = var.local_ssd_count_nvme_block } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml deleted file mode 100644 index 1106f63479..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpx-workload-job.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: batch/v1 -kind: Job -metadata: - name: my-sample-job -spec: - parallelism: 2 - completions: 2 - completionMode: Indexed - template: - spec: - containers: - - name: nccl-test - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpx/nccl-plugin-gpudirecttcpx-dev:v3.1.9 - imagePullPolicy: Always - command: - - /bin/sh - - -c - - | - service ssh restart; - sleep infinity; - env: - - name: LD_LIBRARY_PATH - value: /usr/local/nvidia/lib64 - volumeMounts: - - name: config-volume - mountPath: /configs - resources: - limits: - nvidia.com/gpu: 8 - volumes: - - name: config-volume - configMap: - name: nccl-configmap - defaultMode: 0777 - restartPolicy: Never - backoffLimit: 0 diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml deleted file mode 100644 index 90eea9c902..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/sample-tcpxo-workload-job.yaml +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: batch/v1 -kind: Job -metadata: - name: my-sample-job -spec: - parallelism: 2 - completions: 2 - completionMode: Indexed - template: - spec: - hostname: host1 - subdomain: nccl-host-1 - containers: - - name: nccl-test - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/nccl-plugin-gpudirecttcpx-dev:v1.0.8-1 - imagePullPolicy: Always - command: - - /bin/sh - - -c - - | - set -ex - chmod 755 /scripts/demo-run-nccl-test-tcpxo-via-mpi.sh - cat >/scripts/allgather.sh < 0: - container["env"].extend(env_vars) - container["volumeMounts"].extend(volume_mounts) - -if __name__ == "__main__": - main() diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py deleted file mode 100644 index db9fb3e7ff..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py +++ /dev/null @@ -1,186 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import yaml -import argparse -import os - -def main(): - parser = argparse.ArgumentParser(description="TCPXO Job Manifest Generator") - parser.add_argument("-f", "--file", required=True, help="Path to your job template YAML file") - parser.add_argument("-r", "--rxdm", required=True, help="RxDM version") - - args = parser.parse_args() - - # Get the YAML file from the user - if not args.file: - args.file = input("Please provide the path to your job template YAML file: ") - - # Get component versions from user - if not args.rxdm: - args.rxdm = input("Enter the RxDM version: ") - - # Load and modify the YAML - with open(args.file, "r") as file: - job_manifest = yaml.load(file, Loader=yaml.BaseLoader) - - # Update annotations - add_annotations(job_manifest) - - # Update volumes - add_volumes(job_manifest) - - # Update tolerations - add_tolerations(job_manifest) - - # Add tcpxo-daemon container - add_tcpxo_daemon_container(job_manifest, args.rxdm) - - # Update environment variables and volumeMounts for GPU containers - update_gpu_containers(job_manifest) - - # Generate the new YAML file - updated_job = str(yaml.dump(job_manifest, default_flow_style=False, width=1000, default_style="|", sort_keys=False)).replace("|-", "") - - new_file_name = args.file.replace(".yaml", "-tcpxo.yaml") - with open(new_file_name, "w", encoding="utf-8") as file: - file.write(updated_job) - - # Step 7: Provide instructions to the user - print("\nA new manifest has been generated and updated to have TCPXO enabled based on the provided workload") - print("It can be found in {path}".format(path=os.path.abspath(new_file_name))) - print("You can use the following commands to submit the sample job:") - print(" kubectl create -f {path}".format(path=os.path.abspath(new_file_name))) - -def add_annotations(job_manifest): - annotations = { - 'devices.gke.io/container.tcpxo-daemon':"""|+ -- path: /dev/nvidia0 -- path: /dev/nvidia1 -- path: /dev/nvidia2 -- path: /dev/nvidia3 -- path: /dev/nvidia4 -- path: /dev/nvidia5 -- path: /dev/nvidia6 -- path: /dev/nvidia7 -- path: /dev/nvidiactl -- path: /dev/nvidia-uvm -- path: /dev/dmabuf_import_helper""", - "networking.gke.io/default-interface": "eth0", - "networking.gke.io/interfaces": """| -[ - {"interfaceName":"eth0","network":"default"}, - {"interfaceName":"eth1","network":"vpc1"}, - {"interfaceName":"eth2","network":"vpc2"}, - {"interfaceName":"eth3","network":"vpc3"}, - {"interfaceName":"eth4","network":"vpc4"}, - {"interfaceName":"eth5","network":"vpc5"}, - {"interfaceName":"eth6","network":"vpc6"}, - {"interfaceName":"eth7","network":"vpc7"}, - {"interfaceName":"eth8","network":"vpc8"} -]""", - } - - # Create path if it doesn't exist - job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("metadata", {}) - - # Add/update annotations - pod_template_spec = job_manifest["spec"]["template"]["metadata"] - if "annotations" in pod_template_spec: - pod_template_spec["annotations"].update(annotations) - else: - pod_template_spec["annotations"] = annotations - -def add_tolerations(job_manifest): - tolerations = [ - {"key": "user-workload", "operator": "Equal", "value": """\"true\"""", "effect": "NoSchedule"}, - ] - - # Create path if it doesn't exist - job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) - - # Add tolerations - pod_spec = job_manifest["spec"]["template"]["spec"] - if "tolerations" in pod_spec: - pod_spec["tolerations"].extend(tolerations) - else: - pod_spec["tolerations"] = tolerations - -def add_volumes(job_manifest): - volumes = [ - {"name": "nvidia-install-dir-host", "hostPath": {"path": "/home/kubernetes/bin/nvidia"}}, - {"name": "sys", "hostPath": {"path": "/sys"}}, - {"name": "proc-sys", "hostPath": {"path": "/proc/sys"}}, - {"name": "aperture-devices", "hostPath": {"path": "/dev/aperture_devices"}}, - ] - - # Create path if it doesn't exist - job_manifest.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) - - # Add volumes - pod_spec = job_manifest["spec"]["template"]["spec"] - if "volumes" in pod_spec: - pod_spec["volumes"].extend(volumes) - else: - pod_spec["volumes"] = volumes - - -def add_tcpxo_daemon_container(job_template, rxdm_version): - tcpxo_daemon_container = { - "name": "tcpxo-daemon", - "image": f"us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpxo/tcpgpudmarxd-dev:{rxdm_version}", # Use provided RxDM version - "imagePullPolicy": "Always", - "command": ["/bin/sh", "-c"], - "args": [ - """| - set -ex - chmod 755 /fts/entrypoint_rxdm_container.sh - /fts/entrypoint_rxdm_container.sh --num_hops=2 --num_nics=8 --uid= --alsologtostderr""" - ], - "securityContext": { - "capabilities": {"add": ["NET_ADMIN", "NET_BIND_SERVICE"]} - }, - "volumeMounts": [ - {"name": "nvidia-install-dir-host", "mountPath": "/usr/local/nvidia"}, - {"name": "sys", "mountPath": "/hostsysfs"}, - {"name": "proc-sys", "mountPath": "/hostprocsysfs"}, - ], - "env": [{"name": "LD_LIBRARY_PATH", "value": "/usr/local/nvidia/lib64"}], - } - - # Create path if it doesn't exist - job_template.setdefault("spec", {}).setdefault("template", {}).setdefault("spec", {}) - - # Add container - pod_spec = job_template["spec"]["template"]["spec"] - pod_spec.setdefault("containers", []).insert(0, tcpxo_daemon_container) - -def update_gpu_containers(job_manifest): - env_vars = [ - {"name": "LD_LIBRARY_PATH", "value": "/usr/local/nvidia/lib64"}, - {"name": "NCCL_FASTRAK_LLCM_DEVICE_DIRECTORY", "value": "/dev/aperture_devices"}, - ] - volume_mounts = [{"name": "aperture-devices", "mountPath": "/dev/aperture_devices"}] - - pod_spec = job_manifest.get("spec", {}).get("template", {}).get("spec", {}) - for container in pod_spec.get("containers", []): - # Create path if it doesn't exist - container.setdefault("env", []) - container.setdefault("volumeMounts", []) - if int(container.get("resources", {}).get("limits", {}).get("nvidia.com/gpu", 0)) > 0: - container["env"].extend(env_vars) - container["volumeMounts"].extend(volume_mounts) - -if __name__ == "__main__": - main() diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf deleted file mode 100644 index 002141f800..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/gpu_direct.tf +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -# Enable GPUDirect for A3 and A3Mega VMs, this involve multiple kubectl steps to integrate with the created cluster -# 1. Install NCCL plugin daemonset -# 2. Install NRI plugin daemonset -# 3. Update provided workload to inject rxdm sidecar and other required annotation, volume etc. -locals { - workload_path_tcpx = "${path.module}/gpu-direct-workload/sample-tcpx-workload-job.yaml" - workload_path_tcpxo = "${path.module}/gpu-direct-workload/sample-tcpxo-workload-job.yaml" - - gpu_direct_settings = { - "a3-highgpu-8g" = { - # Manifest to be installed for enabling TCPX on a3-highgpu-8g machines - gpu_direct_manifests = [ - "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/gpudirect-tcpx/nccl-tcpx-installer.yaml", # nccl_plugin v3.1.9 for tcpx - "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/gpudirect-tcpx/nccl-config.yaml", # nccl_configmap - "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/fee883360a660f71ba07478db95d5c1325322f77/nri_device_injector/nri-device-injector.yaml", # nri_plugin - ] - updated_workload_path = replace(local.workload_path_tcpx, ".yaml", "-tcpx.yaml") - rxdm_version = "v2.0.12" # matching nccl-tcpx-installer version v3.1.9 - min_additional_networks = 4 - major_minor_version_acceptable_map = { - "1.27" = "1.27.7-gke.1121000" - "1.28" = "1.28.8-gke.1095000" - "1.29" = "1.29.3-gke.1093000" - "1.30" = "1.30.2-gke.1023000" - } - } - "a3-megagpu-8g" = { - # Manifest to be installed for enabling TCPXO on a3-megagpu-8g machines - gpu_direct_manifests = [ - "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/39308db7574925ea3c14f9113fcf87f70a6fcc26/gpudirect-tcpxo/nccl-tcpxo-installer.yaml", # nccl_plugin v1.0.8-1 for tcpxo - "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/39308db7574925ea3c14f9113fcf87f70a6fcc26/nri_device_injector/nri-device-injector.yaml", # nri_plugin - ] - updated_workload_path = replace(local.workload_path_tcpxo, ".yaml", "-tcpxo.yaml") - rxdm_version = "v1.0.14" # matching nccl-tcpxo-installer version v1.0.8-1 - min_additional_networks = 8 - major_minor_version_acceptable_map = { - "1.28" = "1.28.9-gke.1250000" - "1.29" = "1.29.4-gke.1542000" - "1.30" = "1.30.4-gke.1129000" - "1.31" = "1.31.1-gke.2008000" - "1.32" = "1.32.2-gke.1489001" - } - } - } - - min_additional_networks = try(local.gpu_direct_settings[var.machine_type].min_additional_networks, 0) - - gke_version_regex = "(\\d+\\.\\d+)\\.(\\d+)-gke\\.(\\d+)" # GKE version format: 1.X.Y-gke.Z , regex output: ["1.X" , "Y", "Z"] - - gke_version_parts = regex(local.gke_version_regex, var.gke_version) - gke_version_major = local.gke_version_parts[0] - - major_minor_version_acceptable_map = try(local.gpu_direct_setting[var.machine_type].major_minor_version_acceptable_map, null) - minor_version_acceptable = try(contains(keys(local.major_minor_version_acceptable_map), local.gke_version_major), false) ? local.major_minor_version_acceptable_map[local.gke_version_major] : "1.0.0-gke.0" - minor_version_acceptable_parts = regex(local.gke_version_regex, local.minor_version_acceptable) - gke_gpudirect_compatible = local.gke_version_parts[1] > local.minor_version_acceptable_parts[1] || (local.gke_version_parts[1] == local.minor_version_acceptable_parts[1] && local.gke_version_parts[2] >= local.minor_version_acceptable_parts[2]) -} - -check "gpu_direct_check_multi_vpc" { - assert { - condition = length(var.additional_networks) >= local.min_additional_networks - error_message = "To achieve optimal performance for ${var.machine_type} machine, at least ${local.min_additional_networks} additional vpc is recommended. You could configure it in the blueprint through modules/network/multivpc with network_count set as ${local.min_additional_networks}" - } -} - -check "gke_version_requirements" { - assert { - condition = local.gke_gpudirect_compatible - error_message = "GPUDirect is not supported on GKE version ${var.gke_version} for ${var.machine_type} machine. For supported version details visit https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#requirements" - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf deleted file mode 100644 index 1ddc7ba8c3..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/guest_cpus.tf +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -data "google_compute_machine_types" "machine_info" { - for_each = var.zones == null ? toset([]) : toset(var.zones) - - project = var.project_id - zone = each.key - filter = "name = \"${var.machine_type}\"" -} - -locals { - valid_machine_info = { - for zone, data in data.google_compute_machine_types.machine_info : - zone => data.machine_types if length(data.machine_types) > 0 - } - - guest_cpus = try(local.valid_machine_info[0].guest_cpus, 0) -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/main.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/main.tf deleted file mode 100644 index 857fe4dbc8..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/main.tf +++ /dev/null @@ -1,440 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "gke-node-pool", ghpc_role = "compute" }) -} - -locals { - upgrade_settings = { - strategy = var.upgrade_settings.strategy - max_surge = coalesce(var.upgrade_settings.max_surge, 0) - max_unavailable = coalesce(var.upgrade_settings.max_unavailable, 1) - } -} - -module "gpu" { - source = "../../internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - has_gpu = length(local.guest_accelerator) > 0 - allocatable_gpu_per_node = local.has_gpu ? max(local.guest_accelerator[*].count...) : -1 - is_static_node_pool_with_gpus = var.static_node_count != null && local.allocatable_gpu_per_node != -1 - static_gpu_count = local.is_static_node_pool_with_gpus ? var.static_node_count * local.allocatable_gpu_per_node : 0 - gpu_taint = local.has_gpu ? [{ - key = "nvidia.com/gpu" - value = "present" - effect = "NO_SCHEDULE" - }] : [] - - autoscale_set = var.autoscaling_total_min_nodes != 0 || var.autoscaling_total_max_nodes != 1000 - static_node_set = var.static_node_count != null - initial_node_set = try(var.initial_node_count > 0, false) - - module_unique_id = replace(lower(var.internal_ghpc_module_id), "/[^a-z0-9\\-]/", "") -} - - -locals { - cluster_id_parts = split("/", var.cluster_id) - cluster_name = local.cluster_id_parts[5] - cluster_location = local.cluster_id_parts[3] -} - - -data "google_container_cluster" "gke_cluster" { - name = local.cluster_name - location = local.cluster_location -} - -resource "google_container_node_pool" "node_pool" { - provider = google - - count = max(var.num_node_pools, var.num_slices) - - name = (max(var.num_node_pools, var.num_slices) == 1) ? coalesce(var.name, join("-", [var.machine_type, local.module_unique_id])) : join("-", [coalesce(var.name, join("-", [var.machine_type, local.module_unique_id])), count.index]) - cluster = var.cluster_id - node_locations = var.zones - - node_count = var.static_node_count - dynamic "autoscaling" { - for_each = local.static_node_set ? [] : [1] - content { - total_min_node_count = var.autoscaling_total_min_nodes - total_max_node_count = var.autoscaling_total_max_nodes - location_policy = "ANY" - } - } - - initial_node_count = var.initial_node_count - - max_pods_per_node = var.max_pods_per_node - - management { - auto_repair = var.auto_repair - auto_upgrade = var.auto_upgrade - } - - upgrade_settings { - strategy = local.upgrade_settings.strategy - max_surge = local.upgrade_settings.max_surge - max_unavailable = local.upgrade_settings.max_unavailable - } - - dynamic "placement_policy" { - for_each = var.placement_policy.type != null ? [1] : [] - content { - type = var.placement_policy.type - policy_name = var.placement_policy.name - tpu_topology = var.placement_policy.tpu_topology - } - } - - dynamic "queued_provisioning" { - for_each = var.enable_queued_provisioning ? [1] : [] - content { - enabled = true - } - } - - node_config { - disk_size_gb = var.disk_size_gb - disk_type = var.disk_type - resource_labels = local.labels - labels = var.kubernetes_labels - service_account = var.service_account_email - oauth_scopes = var.service_account_scopes - machine_type = var.machine_type - spot = var.spot - image_type = var.image_type - flex_start = var.enable_flex_start - max_run_duration = var.max_run_duration != null ? "${var.max_run_duration}s" : null - - dynamic "guest_accelerator" { - for_each = local.guest_accelerator - iterator = ga - content { - type = coalesce(ga.value.type, try(local.generated_guest_accelerator[0].type, "")) - count = coalesce(try(ga.value.count, 0) > 0 ? ga.value.count : try(local.generated_guest_accelerator[0].count, "0")) - - gpu_partition_size = try(ga.value.gpu_partition_size, null) - - dynamic "gpu_driver_installation_config" { - # in case user did not specify guest_accelerator settings, we need a try to default to [] - for_each = try([ga.value.gpu_driver_installation_config], [{ gpu_driver_version = "DEFAULT" }]) - iterator = gdic - content { - gpu_driver_version = gdic.value.gpu_driver_version - } - } - - dynamic "gpu_sharing_config" { - for_each = try(ga.value.gpu_sharing_config == null, true) ? [] : [ga.value.gpu_sharing_config] - iterator = gsc - content { - gpu_sharing_strategy = gsc.value.gpu_sharing_strategy - max_shared_clients_per_gpu = gsc.value.max_shared_clients_per_gpu - } - } - } - } - - dynamic "taint" { - for_each = concat(var.taints, local.gpu_taint) - content { - key = taint.value.key - value = taint.value.value - effect = taint.value.effect - } - } - - dynamic "ephemeral_storage_local_ssd_config" { - for_each = local.local_ssd_config.local_ssd_count_ephemeral_storage != null ? [1] : [] - content { - local_ssd_count = local.local_ssd_config.local_ssd_count_ephemeral_storage - } - } - - dynamic "local_nvme_ssd_block_config" { - for_each = local.local_ssd_config.local_ssd_count_nvme_block != null ? [1] : [] - content { - local_ssd_count = local.local_ssd_config.local_ssd_count_nvme_block - } - } - - shielded_instance_config { - enable_secure_boot = var.enable_secure_boot - enable_integrity_monitoring = true - } - - dynamic "gcfs_config" { - for_each = var.enable_gcfs ? [1] : [] - content { - enabled = true - } - } - - gvnic { - enabled = var.image_type == "COS_CONTAINERD" - } - - dynamic "advanced_machine_features" { - for_each = local.set_threads_per_core ? [1] : [] - content { - threads_per_core = local.threads_per_core # relies on threads_per_core_calc.tf - } - } - - # Implied by Workload Identity - workload_metadata_config { - mode = "GKE_METADATA" - } - # Implied by workload identity. - metadata = { - "disable-legacy-endpoints" = "true" - } - - linux_node_config { - sysctls = { - "net.ipv4.tcp_rmem" = "4096 87380 16777216" - "net.ipv4.tcp_wmem" = "4096 16384 16777216" - } - } - - reservation_affinity { - consume_reservation_type = var.reservation_affinity.consume_reservation_type - key = local.is_valid_reservation ? local.reservation_resource_api_label : null - values = local.is_valid_reservation ? (var.is_reservation_active ? local.active_reservation_values : local.default_reservation_values) : null - } - - dynamic "host_maintenance_policy" { - for_each = var.host_maintenance_interval != "" ? [1] : [] - content { - maintenance_interval = var.host_maintenance_interval - } - } - } - - network_config { - dynamic "additional_node_network_configs" { - for_each = var.additional_networks - - content { - network = additional_node_network_configs.value.network - subnetwork = additional_node_network_configs.value.subnetwork - } - } - - enable_private_nodes = var.enable_private_nodes - } - - timeouts { - create = var.timeout_create - update = var.timeout_update - } - - lifecycle { - ignore_changes = [ - node_config[0].labels, - initial_node_count, - # Ignore local/ephemeral ssd configs as they are tied to machine types. - node_config[0].ephemeral_storage_local_ssd_config, - node_config[0].local_nvme_ssd_block_config, - ] - precondition { - condition = (var.max_pods_per_node == null) || (data.google_container_cluster.gke_cluster.networking_mode == "VPC_NATIVE") - error_message = "max_pods_per_node does not work on `routes-based` clusters, that don't have IP Aliasing enabled." - } - precondition { - condition = !local.static_node_set || !local.autoscale_set - error_message = "static_node_count cannot be set with either autoscaling_total_min_nodes or autoscaling_total_max_nodes." - } - precondition { - condition = !local.static_node_set || !local.initial_node_set - error_message = "initial_node_count cannot be set with static_node_count." - } - precondition { - condition = !local.initial_node_set || (coalesce(var.initial_node_count, 0) >= var.autoscaling_total_min_nodes && coalesce(var.initial_node_count, 0) <= var.autoscaling_total_max_nodes) - error_message = "initial_node_count must be between autoscaling_total_min_nodes and autoscaling_total_max_nodes included." - } - precondition { - condition = !(coalesce(local.local_ssd_config.local_ssd_count_ephemeral_storage, 0) > 0 && coalesce(local.local_ssd_config.local_ssd_count_nvme_block, 0) > 0) - error_message = "Only one of local_ssd_count_ephemeral_storage or local_ssd_count_nvme_block can be set to a non-zero value." - } - precondition { - condition = ( - (var.reservation_affinity.consume_reservation_type != "SPECIFIC_RESERVATION" && local.input_specific_reservations_count == 0) || - (var.reservation_affinity.consume_reservation_type == "SPECIFIC_RESERVATION" && local.input_specific_reservations_count == 1) - ) - error_message = <<-EOT - When using NO_RESERVATION or ANY_RESERVATION as the `consume_reservation_type`, `specific_reservations` cannot be set. - On the other hand, with SPECIFIC_RESERVATION you must set `specific_reservations`. - EOT - } - precondition { - condition = ( - (local.input_specific_reservations_count == 0) || - ((length(local.verified_specific_reservations) == 1 || !var.is_reservation_active) && - length(local.specific_reservation_requirement_violations) == 0) - ) - error_message = <<-EOT - Check if your reservation is configured correctly: - - A reservation with the name must exist in the specified project and one of the specified zones - - - Its consumption type must be "specific" - %{for property in local.specific_reservation_requirement_violations} - - ${local.specific_reservation_requirement_violation_messages[property]} - %{endfor} - EOT - } - precondition { - condition = ( - (local.input_specific_reservations_count == 0) || - (local.input_specific_reservations_count == 1 && length(local.input_reservation_suffixes) == 0) || - (local.input_specific_reservations_count == 1 && length(local.input_reservation_suffixes) > 0 && try(local.input_reservation_projects[0], var.project_id) == var.project_id) - ) - error_message = "Shared extended reservations are not supported by GKE." - } - precondition { - condition = contains(["SURGE"], local.upgrade_settings.strategy) - error_message = "Only SURGE strategy is supported" - } - precondition { - condition = local.upgrade_settings.max_unavailable >= 0 - error_message = "max_unavailable should be set to 0 or greater" - } - precondition { - condition = local.upgrade_settings.max_surge >= 0 - error_message = "max_surge should be set to 0 or greater" - } - precondition { - condition = local.upgrade_settings.max_unavailable > 0 || local.upgrade_settings.max_surge > 0 - error_message = "At least one of max_unavailable or max_surge must greater than 0" - } - precondition { - condition = var.placement_policy.type != "COMPACT" || (var.zones != null ? (length(var.zones) == 1) : false) - error_message = "Compact placement is only available for node pools operating in a single zone." - } - precondition { - condition = var.placement_policy.type != "COMPACT" || local.upgrade_settings.strategy != "BLUE_GREEN" - error_message = "Compact placement is not supported with blue-green upgrades." - } - precondition { - condition = !(var.enable_queued_provisioning == true && var.placement_policy.type == "COMPACT") - error_message = "placement_policy cannot be COMPACT when enable_queued_provisioning is true." - } - precondition { - condition = !(var.enable_queued_provisioning == true && var.reservation_affinity.consume_reservation_type != "NO_RESERVATION") - error_message = "reservation_affinity should be NO_RESERVATION when enable_queued_provisioning is true." - } - precondition { - condition = !(var.enable_queued_provisioning == true && var.autoscaling_total_min_nodes != 0) - error_message = "autoscaling_total_min_nodes should be 0 when enable_queued_provisioning is true." - } - precondition { - condition = !(var.num_node_pools > 1 && var.num_slices > 1) - error_message = "num_node_pools is for CPUs and GPUS, and num_slices is for TPUs. Both cannot be set at the same time to create a group of identical nodepools / slices." - } - precondition { - condition = !(var.num_node_pools == 0 && var.num_slices == 0) - error_message = "Either num_node_pools (for CPUs and GPUS) or num_slices (for TPUs) should be set to a positive integer value." - } - precondition { - condition = !(var.num_node_pools < 0 || var.num_slices < 0) - error_message = "Negative integer value of num_node_pools or num_slices is not valid. Please use a positive integer value to set num_node_pools for CPUs and GPUS, and num_slices for TPUs." - } - precondition { - condition = var.enable_flex_start == true ? (var.auto_repair == false) : true - error_message = "enable_flex_start needs node auto_repair set to false." - } - precondition { - condition = var.enable_flex_start == true ? (var.static_node_count == null) : true - error_message = "enable_flex_start does not work with static_node_count. static_node_count should be set to null." - } - precondition { - condition = var.enable_flex_start == true ? (var.reservation_affinity.consume_reservation_type == "NO_RESERVATION") : true - error_message = "enable_flex_start only works with reservation_affinity consume_reservation_type NO_RESERVATION." - } - precondition { - condition = var.enable_flex_start == true ? (var.spot == false) : true - error_message = "Both enable_flex_start and spot consumption option cannot be set to true at the same time." - } - } -} - -locals { - supported_machine_types_for_install_dependencies = ["a3-highgpu-8g", "a3-megagpu-8g"] -} - -resource "null_resource" "install_dependencies" { - count = var.run_workload_script && contains(local.supported_machine_types_for_install_dependencies, var.machine_type) ? 1 : 0 - provisioner "local-exec" { - command = "pip3 install pyyaml" - } -} - -locals { - gpu_direct_setting = lookup(local.gpu_direct_settings, var.machine_type, { gpu_direct_manifests = [], updated_workload_path = "", rxdm_version = "" }) -} - -# execute script to inject rxdm sidecar into workload to enable tcpx for a3-highgpu-8g VM workload -resource "null_resource" "enable_tcpx_in_workload" { - count = var.run_workload_script && var.machine_type == "a3-highgpu-8g" ? 1 : 0 - triggers = { - always_run = timestamp() - } - provisioner "local-exec" { - command = "python3 ${path.module}/gpu-direct-workload/scripts/enable-tcpx-in-workload.py --file ${local.workload_path_tcpx} --rxdm ${local.gpu_direct_setting.rxdm_version}" - } - - depends_on = [null_resource.install_dependencies] -} - -# execute script to inject rxdm sidecar into workload to enable tcpxo for a3-megagpu-8g VM workload -resource "null_resource" "enable_tcpxo_in_workload" { - count = var.run_workload_script && var.machine_type == "a3-megagpu-8g" ? 1 : 0 - triggers = { - always_run = timestamp() - } - provisioner "local-exec" { - command = "python3 ${path.module}/gpu-direct-workload/scripts/enable-tcpxo-in-workload.py --file ${local.workload_path_tcpxo} --rxdm ${local.gpu_direct_setting.rxdm_version}" - } - - depends_on = [null_resource.install_dependencies] -} - -# apply manifest to enable tcpx -module "kubectl_apply" { - source = "../../management/kubectl-apply" - - cluster_id = var.cluster_id - project_id = var.project_id - - apply_manifests = flatten([ - for manifest in local.gpu_direct_setting.gpu_direct_manifests : [ - { - source = manifest - } - ] - ]) -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/metadata.yaml deleted file mode 100644 index e980d595a2..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com -ghpc: - inject_module_id: internal_ghpc_module_id diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/outputs.tf deleted file mode 100644 index aa88c7d5bc..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/outputs.tf +++ /dev/null @@ -1,112 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "node_pool_names" { - description = "Names of the node pools." - value = google_container_node_pool.node_pool[*].name -} - -locals { - # Shared core machines only have 1 cpu allocatable, even if they have 2 cpu capacity - vcpu = local.machine_shared_core ? 1 : local.guest_cpus - useable_cpu = local.set_threads_per_core ? local.threads_per_core * local.vcpu / 2 : local.vcpu - - # allocatable resource definition: https://cloud.google.com/kubernetes-engine/docs/concepts/plan-node-sizes#cpu_reservations - second_core = local.useable_cpu > 1 ? 1 : 0 - third_fourth_core = local.useable_cpu == 3 ? 1 : local.useable_cpu > 3 ? 2 : 0 - cores_above_four = local.useable_cpu > 4 ? local.useable_cpu - 4 : 0 - - allocatable_cpu = 0.94 + (0.99 * local.second_core) + (0.995 * local.third_fourth_core) + (0.9975 * local.cores_above_four) -} - -output "allocatable_cpu_per_node" { - description = "Number of CPUs available for scheduling pods on each node." - value = local.allocatable_cpu -} - -output "has_gpu" { - description = "Boolean value indicating whether nodes in the pool are configured with GPUs." - value = local.has_gpu -} - -output "allocatable_gpu_per_node" { - description = "Number of GPUs available for scheduling pods on each node." - value = local.allocatable_gpu_per_node -} - -output "static_gpu_count" { - description = "Total number of GPUs in the node pool. Available only for static node pools." - value = local.static_gpu_count -} - -locals { - translate_toleration = { - PREFER_NO_SCHEDULE = "PreferNoSchedule" - NO_SCHEDULE = "NoSchedule" - NO_EXECUTE = "NoExecute" - } - taints = google_container_node_pool.node_pool[0].node_config[0].taint - tolerations = [for taint in local.taints : { - key = taint.key - operator = "Equal" - value = taint.value - effect = lookup(local.translate_toleration, taint.effect, null) - }] -} - -output "tolerations" { - description = "Tolerations needed for a pod to be scheduled on this node pool." - value = local.tolerations -} - -locals { - gpu_direct_enabled = var.machine_type == "a3-highgpu-8g" || var.machine_type == "a3-megagpu-8g" - script_path = { - a3-highgpu-8g = "enable-tcpx-in-workload.py", - a3-megagpu-8g = "enable-tcpxo-in-workload.py" - } - nccl_path = var.machine_type == "a3-highgpu-8g" ? "configs" : "scripts" - gpu_direct_instruction = <<-EOT - Since you are using ${var.machine_type} machine type that has GPUDirect support, your nodepool had been configured with the required plugins. - To fully utilize GPUDirect you will need to add some components into your workload manifest. Details below: - - A sample GKE job that has GPUDirect enabled and NCCL test included has been generated locally at: - ${abspath(local.gpu_direct_setting.updated_workload_path)} - - You can use the following commands to submit the sample job: - kubectl create -f ${abspath(local.gpu_direct_setting.updated_workload_path)} - After submitting the sample job, you can validate the GPU performance by initiating NCCL test included in the sample workload: - NCCL test can be initiated from any one of the sample job Pods and coordinate with the peer Pods: - export POD_NAME=$(kubectl get pods -l job-name=my-sample-job -o go-template='{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' | head -n 1) - export PEER_POD_IPS=$(kubectl get pods -l job-name=my-sample-job -o go-template='{{range .items}}{{.status.podIP}}{{" "}}{{end}}') - kubectl exec --stdin --tty --container=nccl-test $POD_NAME -- /${local.nccl_path}/allgather.sh $PEER_POD_IPS - - If you would like to enable GPUDirect for your own workload, please follow the below steps: - export WORKLOAD_PATH=<> - python3 ${abspath("${path.module}/gpu-direct-workload/scripts/${lookup(local.script_path, var.machine_type, "")}")} --file $WORKLOAD_PATH --rxdm ${local.gpu_direct_setting.rxdm_version} - **WARNING** - The "--rxdm" version is tied to the nccl-tcpx/o-installer that had been deployed to your cluster, changing it to other value might have impact on performance - **WARNING** - - Or you can also follow our GPUDirect user guide to update your workload - https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#add-gpudirect-manifests - EOT -} - -output "instructions" { - description = "Instructions for submitting the sample GPUDirect enabled job." - value = local.gpu_direct_enabled ? local.gpu_direct_instruction : null -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf deleted file mode 100644 index 359e7c5aa8..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/reservation_definitions.tf +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -# Split the input into three different lists where the details of a given reservation are at the same index across these lists. -locals { - # Specific block of an extended reservation can be targeted with exr-one/reservationBlocks/exr-one-block-1 - # Data source needs to be queried with the reservation name only. So, we extract the reservation name - input_reservation_names = [for r in try(var.reservation_affinity.specific_reservations, []) : split("/", r.name)[0]] - input_reservation_projects = [for r in try(var.reservation_affinity.specific_reservations, []) : coalesce(r.project, var.project_id)] - # We, also, remember the suffix "/reservationBlocks/exr-one-block-1" for use elsewhere afterwards - input_reservation_suffixes = [for r in try(var.reservation_affinity.specific_reservations, []) : substr(r.name, length(split("/", r.name)[0]), -1)] -} - -data "google_compute_reservation" "specific_reservations" { - for_each = ( - local.input_specific_reservations_count == 0 ? - {} : - { - for pair in flatten([ - for zone in try(var.zones, []) : [ - for i, reservation_name in try(local.input_reservation_names, []) : { - key : "${local.input_reservation_projects[i]}/${zone}/${reservation_name}" - zone : zone - reservation_name : reservation_name - project : local.input_reservation_projects[i] - } - ] - ]) : - pair.key => pair - } - ) - name = each.value.reservation_name - zone = each.value.zone - project = each.value.project -} - -locals { - generated_guest_accelerator = module.gpu.machine_type_guest_accelerator - reservation_resource_api_label = "compute.googleapis.com/reservation-name" - input_specific_reservations_count = try(length(var.reservation_affinity.specific_reservations), 0) - - # Filter specific reservations - verified_specific_reservations = [for k, v in data.google_compute_reservation.specific_reservations : v if(v.specific_reservation != null && v.specific_reservation_required == true)] - - # Build two maps to be used to compare the VM properties between reservations and the node pool - # Skip this for TPUs - reservation_vm_properties = local.has_gpu ? ([for reservation in local.verified_specific_reservations : { - "machine_type" : try(reservation.specific_reservation[0].instance_properties[0].machine_type, "") - "guest_accelerators" : { for acc in try(reservation.specific_reservation[0].instance_properties[0].guest_accelerators, []) : acc.accelerator_type => acc.accelerator_count } - }]) : [] - nodepool_vm_properties = { - "machine_type" : var.machine_type - "guest_accelerators" : { for acc in try(local.guest_accelerator, []) : coalesce(acc.type, try(local.generated_guest_accelerator[0].type, "")) => coalesce(acc.count, try(local.generated_guest_accelerator[0].count, 0)) } - } - - # Compare two maps by counting the keys that mismatch. - # Know that in map comparison the order of keys does not matter. That is {NVME: x, SCSI: y} and {SCSI: y, NVME: x} are equal - # As of this writing, there is only one reservation supported by the Node Pool API. So, directly accessing it from the list - specific_reservation_requirement_violations = length(local.reservation_vm_properties) == 0 ? [] : [for k, v in local.nodepool_vm_properties : k if v != local.reservation_vm_properties[0][k]] - - specific_reservation_requirement_violation_messages = { - "machine_type" : <<-EOT - The reservation has "${try(local.reservation_vm_properties[0].machine_type, "")}" machine type and the node pool has "${local.nodepool_vm_properties.machine_type}". Check the relevant node pool setting: "machine_type" - EOT - "guest_accelerators" : <<-EOT - The reservation has ${jsonencode(try(local.reservation_vm_properties[0].guest_accelerators, {}))} accelerators and the node pool has ${jsonencode(try(local.nodepool_vm_properties.guest_accelerators, {}))}. Check the relevant node pool setting: "guest_accelerator". When unspecified, for the machine_type=${var.machine_type}, the default is guest_accelerator=${jsonencode(try(local.generated_guest_accelerator, [{}]))}. - EOT - } -} - -locals { - # Check if reservation is valid, that is, if it exists, there should be only 1 verified specific reservation or the reservation doesn't exist - is_valid_reservation = length(local.verified_specific_reservations) == 1 || !var.is_reservation_active - - # Build the list of reservation names when var.is_reservation_active is true - active_reservation_values = [ - for i, r in local.verified_specific_reservations : - length(local.input_reservation_suffixes[i]) > 0 ? - format("%s%s", r.name, local.input_reservation_suffixes[i]) : - "projects/${r.project}/reservations/${r.name}" - ] - - # Define a default reservation value if no specific reservations are present - specific_reservation_name = length(local.input_reservation_names) > 0 ? local.input_reservation_names[0] : "" - default_reservation_values = ["projects/${var.project_id}/reservations/${local.specific_reservation_name}"] -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf deleted file mode 100644 index e582db33da..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/threads_per_core_calc.tf +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# This file is meant to be reused by multiple modules. -# "description": Allows for 'threads_per_core=0: SMT will be disabled where compatible (default)' - -# "inputs": -# var.machine_type: Machine type for the instance being evaluated. -# var.threads_per_core : Sets the number of threads per physical core, where 0 -# has behavior described in description. - -# "outputs": -# local.set_threads_per_core: bool that tells if threads per core should be set, -# to be used with a dynamic block. -# local.threads_per_core: actual threads_per_core to be used. - -locals { - machine_vals = split("-", var.machine_type) - machine_family = local.machine_vals[0] - machine_shared_core = length(local.machine_vals) <= 2 - machine_vcpus = try(parseint(local.machine_vals[2], 10), 1) - - smt_capable_family = !contains(["t2d", "t2a"], local.machine_family) - smt_capable_vcpu = local.machine_vcpus >= 2 - - smt_capable = local.smt_capable_family && local.smt_capable_vcpu && !local.machine_shared_core - set_threads_per_core = var.threads_per_core != null && (var.threads_per_core == 0 && local.smt_capable || try(var.threads_per_core >= 1, false)) - threads_per_core = var.threads_per_core == 2 ? 2 : 1 -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/variables.tf deleted file mode 100644 index cfd1dcd6b0..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/variables.tf +++ /dev/null @@ -1,472 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "The project ID to host the cluster in." - type = string -} - -variable "cluster_id" { - description = "projects/{{project}}/locations/{{location}}/clusters/{{cluster}}" - type = string -} - -variable "zones" { - description = "A list of zones to be used. Zones must be in region of cluster. If null, cluster zones will be inherited. Note `zones` not `zone`; does not work with `zone` deployment variable." - type = list(string) - default = null -} - -variable "name" { - description = <<-EOD - The name of the node pool. If not set, automatically populated by machine type and module id (unique blueprint-wide) as suffix. - If setting manually, ensure a unique value across all gke-node-pools. - EOD - type = string - default = null -} - -variable "internal_ghpc_module_id" { - description = "DO NOT SET THIS MANUALLY. Automatically populates with module id (unique blueprint-wide)." - type = string -} - -variable "machine_type" { - description = "The name of a Google Compute Engine machine type." - type = string - default = "c2-standard-60" -} - -variable "disk_size_gb" { - description = "Size of disk for each node." - type = number - default = 100 -} - -variable "disk_type" { - description = "Disk type for each node." - type = string - default = null -} - -variable "enable_gcfs" { - description = "Enable the Google Container Filesystem (GCFS). See [restrictions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#gcfs_config)." - type = bool - default = false -} - -variable "enable_secure_boot" { - description = "Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info." - type = bool - default = true -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = optional(string) - count = optional(number, 0) - gpu_driver_installation_config = optional(object({ - gpu_driver_version = string - }), { gpu_driver_version = "DEFAULT" }) - gpu_partition_size = optional(string) - gpu_sharing_config = optional(object({ - gpu_sharing_strategy = string - max_shared_clients_per_gpu = number - })) - })) - default = [] - nullable = false - - validation { - condition = alltrue([for ga in var.guest_accelerator : ga.count != null]) - error_message = "var.guest_accelerator[*].count cannot be null" - } - - validation { - condition = alltrue([for ga in var.guest_accelerator : ga.count >= 0]) - error_message = "var.guest_accelerator[*].count must never be negative" - } - - validation { - condition = alltrue([for ga in var.guest_accelerator : ga.gpu_driver_installation_config != null]) - error_message = "var.guest_accelerator[*].gpu_driver_installation_config must not be null; leave unset to enable GKE to select default GPU driver installation" - } -} - -variable "image_type" { - description = "The default image type used by NAP once a new node pool is being created. Use either COS_CONTAINERD or UBUNTU_CONTAINERD." - type = string - default = "COS_CONTAINERD" -} - -variable "local_ssd_count_ephemeral_storage" { - description = <<-EOT - The number of local SSDs to attach to each node to back ephemeral storage. - Uses NVMe interfaces. Must be supported by `machine_type`. - When set to null, default value either is [set based on machine_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value. - [See above](#local-ssd-storage) for more info. - EOT - type = number - default = null -} - -variable "local_ssd_count_nvme_block" { - description = <<-EOT - The number of local SSDs to attach to each node to back block storage. - Uses NVMe interfaces. Must be supported by `machine_type`. - When set to null, default value either is [set based on machine_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value. - [See above](#local-ssd-storage) for more info. - - EOT - type = number - default = null -} - -variable "autoscaling_total_min_nodes" { - description = "Total minimum number of nodes in the NodePool." - type = number - default = 0 -} - -variable "autoscaling_total_max_nodes" { - description = "Total maximum number of nodes in the NodePool." - type = number - default = 1000 -} - -variable "static_node_count" { - description = "The static number of nodes in the node pool. If set, autoscaling will be disabled." - type = number - default = null -} - -variable "is_reservation_active" { - description = "Whether the specified reservation is already created." - type = bool - default = true -} - -variable "auto_repair" { - description = "Whether the nodes will be automatically repaired." - type = bool - default = true -} - -variable "auto_upgrade" { - description = "Whether the nodes will be automatically upgraded." - type = bool - default = false -} - -variable "threads_per_core" { - description = <<-EOT - Sets the number of threads per physical core. By setting threads_per_core - to 2, Simultaneous Multithreading (SMT) is enabled extending the total number - of virtual cores. For example, a machine of type c2-standard-60 will have 60 - virtual cores with threads_per_core equal to 2. With threads_per_core equal - to 1 (SMT turned off), only the 30 physical cores will be available on the VM. - - The default value of \"0\" will turn off SMT for supported machine types, and - will fall back to GCE defaults for unsupported machine types (t2d, shared-core - instances, or instances with less than 2 vCPU). - - Disabling SMT can be more performant in many HPC workloads, therefore it is - disabled by default where compatible. - - null = SMT configuration will use the GCE defaults for the machine type - 0 = SMT will be disabled where compatible (default) - 1 = SMT will always be disabled (will fail on incompatible machine types) - 2 = SMT will always be enabled (will fail on incompatible machine types) - EOT - type = number - default = 0 - - validation { - condition = var.threads_per_core == null || try(var.threads_per_core >= 0, false) && try(var.threads_per_core <= 2, false) - error_message = "Allowed values for threads_per_core are \"null\", \"0\", \"1\", \"2\"." - } -} - -variable "spot" { - description = "Provision VMs using discounted Spot pricing, allowing for preemption" - type = bool - default = false -} - -# tflint-ignore: terraform_unused_declarations -variable "compact_placement" { - description = "DEPRECATED: Use `placement_policy`" - type = bool - default = null - validation { - condition = var.compact_placement == null - error_message = "`compact_placement` is deprecated. Use `placement_policy` instead" - } -} - -variable "placement_policy" { - description = <<-EOT - Group placement policy to use for the node pool's nodes. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. `tpu_topology` is the TPU placement topology for pod slice node pool. - It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement. - Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. - EOT - - type = object({ - type = string - name = optional(string) - tpu_topology = optional(string) - }) - default = { - type = null - name = null - tpu_topology = null - } - validation { - condition = var.placement_policy.type == null || try(contains(["COMPACT"], var.placement_policy.type), false) - error_message = "`COMPACT` is the only supported value for `placement_policy.type`." - } -} - -variable "service_account_email" { - description = "Service account e-mail address to use with the node pool" - type = string - default = null -} - -variable "service_account_scopes" { - description = "Scopes to to use with the node pool." - type = set(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -variable "taints" { - description = "Taints to be applied to the system node pool." - type = list(object({ - key = string - value = any - effect = string - })) - default = [] -} - -variable "labels" { - description = "GCE resource labels to be applied to resources. Key-value pairs." - type = map(string) -} - -variable "kubernetes_labels" { - description = <<-EOT - Kubernetes labels to be applied to each node in the node group. Key-value pairs. - (The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) - EOT - type = map(string) - default = null -} - -variable "timeout_create" { - description = "Timeout for creating a node pool" - type = string - default = null -} - -variable "timeout_update" { - description = "Timeout for updating a node pool" - type = string - default = null -} - -# Deprecated - -# tflint-ignore: terraform_unused_declarations -variable "total_min_nodes" { - description = "DEPRECATED: Use autoscaling_total_min_nodes." - type = number - default = null - validation { - condition = var.total_min_nodes == null - error_message = "total_min_nodes was renamed to autoscaling_total_min_nodes and is deprecated; use autoscaling_total_min_nodes" - } -} - -# tflint-ignore: terraform_unused_declarations -variable "total_max_nodes" { - description = "DEPRECATED: Use autoscaling_total_max_nodes." - type = number - default = null - validation { - condition = var.total_max_nodes == null - error_message = "total_max_nodes was renamed to autoscaling_total_max_nodes and is deprecated; use autoscaling_total_max_nodes" - } -} - -# tflint-ignore: terraform_unused_declarations -variable "service_account" { - description = "DEPRECATED: use service_account_email and scopes." - type = object({ - email = string, - scopes = set(string) - }) - default = null - validation { - condition = var.service_account == null - error_message = "service_account is deprecated and replaced with service_account_email and scopes." - } -} - -variable "additional_networks" { - description = "Additional network interface details for GKE, if any. Providing additional networks adds additional node networks to the node pool" - default = [] - type = list(object({ - network = string - subnetwork = string - subnetwork_project = string - network_ip = string - nic_type = string - stack_type = string - queue_count = number - access_config = list(object({ - nat_ip = string - network_tier = string - })) - ipv6_access_config = list(object({ - network_tier = string - })) - alias_ip_range = list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })) - })) - nullable = false -} - -variable "reservation_affinity" { - description = <<-EOT - Reservation resource to consume. When targeting SPECIFIC_RESERVATION, specific_reservations needs be specified. - Even though specific_reservations is a list, only one reservation is allowed by the NodePool API. - It is assumed that the specified reservation exists and has available capacity. - For a shared reservation, specify the project_id as well in which it was created. - To create a reservation refer to https://cloud.google.com/compute/docs/instances/reservations-single-project and https://cloud.google.com/compute/docs/instances/reservations-shared - EOT - type = object({ - consume_reservation_type = string - specific_reservations = optional(list(object({ - name = string - project = optional(string) - }))) - }) - default = { - consume_reservation_type = "NO_RESERVATION" - specific_reservations = [] - } - validation { - condition = contains(["NO_RESERVATION", "ANY_RESERVATION", "SPECIFIC_RESERVATION"], var.reservation_affinity.consume_reservation_type) - error_message = "Accepted values are: {NO_RESERVATION, ANY_RESERVATION, SPECIFIC_RESERVATION}" - } -} - -variable "host_maintenance_interval" { - description = "Specifies the frequency of planned maintenance events." - type = string - default = "" - nullable = false - validation { - condition = contains(["", "PERIODIC", "AS_NEEDED"], var.host_maintenance_interval) - error_message = "Invalid host_maintenance_interval value. Must be PERIODIC, AS_NEEDED or the empty string" - } -} - -variable "initial_node_count" { - description = "The initial number of nodes for the pool. In regional clusters, this is the number of nodes per zone. Changing this setting after node pool creation will not make any effect. It cannot be set with static_node_count and must be set to a value between autoscaling_total_min_nodes and autoscaling_total_max_nodes." - type = number - default = null -} - -variable "gke_version" { - description = "GKE version" - type = string -} - -variable "max_pods_per_node" { - description = "The maximum number of pods per node in this node pool. This will force replacement." - type = number - default = null -} - -variable "upgrade_settings" { - description = <<-EOT - Defines node pool upgrade settings. It is highly recommended that you define all max_surge and max_unavailable. - If max_surge is not specified, it would be set to a default value of 0. - If max_unavailable is not specified, it would be set to a default value of 1. - EOT - type = object({ - strategy = string - max_surge = optional(number) - max_unavailable = optional(number) - }) - default = { - strategy = "SURGE" - max_surge = 0 - max_unavailable = 1 - } -} - -variable "run_workload_script" { - description = "Whether execute the script to create a sample workload and inject rxdm sidecar into workload. Currently, implemented for A3-Highgpu and A3-Megagpu only." - type = bool - default = true -} - -variable "enable_queued_provisioning" { - description = "If true, enables Dynamic Workload Scheduler and adds the cloud.google.com/gke-queued taint to the node pool." - type = bool - default = false -} - -variable "enable_flex_start" { - description = <<-EOT - If true, start the node pool with Flex Start provisioning model. - To learn more about flex-start mode, please refer to - https://cloud.google.com/kubernetes-engine/docs/how-to/dws-flex-start-training and - https://cloud.google.com/kubernetes-engine/docs/how-to/provisioningrequest - EOT - type = bool - default = false -} - -variable "max_run_duration" { - description = "The duration (in whole seconds) of the instance. Instance will run and be terminated after then." - type = number - default = null -} - -variable "enable_private_nodes" { - description = "Whether nodes have internal IP addresses only." - type = bool - default = false -} - -variable "num_node_pools" { - description = "Number of node pools to create. This is same as num_slices." - type = number - default = 1 -} - -variable "num_slices" { - description = "Number of TPUs slices to create. This is same as num_node_pools." - type = number - default = 1 -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/versions.tf deleted file mode 100644 index 26df685e61..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/gke-node-pool/versions.tf +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.5" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.41" - } - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.57.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:gke-node-pool/v1.45.0" - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/README.md b/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/README.md deleted file mode 100644 index d91a923310..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/README.md +++ /dev/null @@ -1,82 +0,0 @@ -## Description - -This modules create a [resource policy for compute engines](https://cloud.google.com/compute/docs/instances/placement-policies-overview). This policy can be passed to a gke-node-pool module to apply the policy on the node-pool's nodes. - -Note: By default, you can't apply compact placement policies with a max distance value to A3 VMs. To request access to this feature, contact your [Technical Account Manager (TAM)](https://cloud.google.com/tam) or the [Sales team](https://cloud.google.com/contact). - -### Example - -The following example creates a group placement resource policy and applies it to a gke-node-pool. - -```yaml - - id: group_placement_1 - source: modules/compute/resource-policy - settings: - name: gp-np-1 - group_placement_max_distance: 2 - - - id: node_pool_1 - source: modules/compute/gke-node-pool - use: [group_placement_1] - settings: - machine_type: e2-standard-8 - outputs: [instructions] -``` - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google-beta](#requirement\_google-beta) | >= 6.29.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google-beta](#provider\_google-beta) | >= 6.29.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_resource_policy.policy](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_resource_policy) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [group\_placement\_max\_distance](#input\_group\_placement\_max\_distance) | The max distance for group placement policy to use for the node pool's nodes. If set it will add a compact group placement policy.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. | `number` | `0` | no | -| [name](#input\_name) | The resource policy's name. | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | The project ID for the resource policy. | `string` | n/a | yes | -| [region](#input\_region) | The region for the the resource policy. | `string` | n/a | yes | -| [workload\_policy](#input\_workload\_policy) | Describes the workload policy | 
object({
    type                  = optional(string, null)
    max_topology_distance = optional(string, null)
    accelerator_topology  = optional(string, null)
  })
| 
{
  "accelerator_topology": null,
  "max_topology_distance": null,
  "type": null
}
| no | - -## Outputs - -| Name | Description | -|------|-------------| -| [placement\_policy](#output\_placement\_policy) | Group placement policy to use for placing VMs or GKE nodes placement. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy.
It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/main.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/main.tf deleted file mode 100644 index 906424ca7c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/main.tf +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -locals { - name = "${var.name}-${random_id.resource_name_suffix.hex}" -} - -resource "google_compute_resource_policy" "policy" { - name = local.name - region = var.region - project = var.project_id - provider = google-beta - - dynamic "workload_policy" { - for_each = var.workload_policy.type != null ? [1] : [] - - content { - type = var.workload_policy.type - max_topology_distance = var.workload_policy.max_topology_distance - accelerator_topology = var.workload_policy.accelerator_topology - } - } - - dynamic "group_placement_policy" { - for_each = var.group_placement_max_distance > 0 ? [1] : [] - - content { - collocation = "COLLOCATED" - max_distance = var.group_placement_max_distance - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/outputs.tf deleted file mode 100644 index b4b2a4e1b3..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/outputs.tf +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "placement_policy" { - description = <<-EOT - Group placement policy to use for placing VMs or GKE nodes placement. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy. - It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement. - Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. - EOT - - value = { - type = (var.group_placement_max_distance > 0 || var.workload_policy.type != null) ? "COMPACT" : null - name = (var.group_placement_max_distance > 0 || var.workload_policy.type != null) ? local.name : null - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/variables.tf deleted file mode 100644 index e70ecca8a0..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/variables.tf +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "The project ID for the resource policy." - type = string -} - -variable "region" { - description = "The region for the the resource policy." - type = string -} - -variable "name" { - description = "The resource policy's name." - type = string -} - -variable "group_placement_max_distance" { - description = <<-EOT - The max distance for group placement policy to use for the node pool's nodes. If set it will add a compact group placement policy. - Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. - EOT - - type = number - default = 0 -} - -variable "workload_policy" { - description = "Describes the workload policy" - type = object({ - type = optional(string, null) - max_topology_distance = optional(string, null) - accelerator_topology = optional(string, null) - }) - default = { - type = null - max_topology_distance = null - accelerator_topology = null - } - nullable = false -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/versions.tf deleted file mode 100644 index f235fbade3..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/resource-policy/versions.tf +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google-beta = { - source = "hashicorp/google-beta" - version = ">= 6.29.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:resource-policy/v1.37.2" - } - - required_version = ">= 1.3" -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/README.md b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/README.md deleted file mode 100644 index c2535ee03e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/README.md +++ /dev/null @@ -1,258 +0,0 @@ -## Description - -This module creates one or more -[compute VM instances](https://cloud.google.com/compute/docs/instances). - -### Example - -```yaml -- id: compute - source: modules/compute/vm-instance - use: [network1] - settings: - instance_count: 8 - name_prefix: compute - machine_type: c2-standard-60 -``` - -This creates a cluster of 8 compute VMs that are: - -* named `compute-[0-7]` -* on the network defined by the `network1` module -* of type c2-standard-60 - -> **_NOTE:_** Simultaneous Multithreading (SMT) is deactivated by default -> (threads_per_core=1), which means only the physical cores are visible on the -> VM. With SMT disabled, a machine of type c2-standard-60 will only have the 30 -> physical cores visible. To change this, set `threads_per_core=2` under -> settings. - -### VPC Networks - -There are two methods for adding network connectivity to the `vm-instance` -module. The first is shown in the example above, where a `vpc` module or -`pre-existing-vpc` module is used by the `vm-instance` module. When this -happens, the `network_self_link` and `subnetwork_self_link` outputs from the -network are provided as input to the `vm-instance` and a network interface is -defined based on that. This can also be done updating the `network_self_link` and -`subnetwork_self_link` settings directly. - -The alternative option can be used when more than one network needs to be added -to the `vm-instance` or further customization is needed beyond what is provided -via other variables. For this option, the `network_interfaces` variable can be -used to set up one or more network interfaces on the VM instance. The format is -consistent with the terraform `google_compute_instance` `network_interface` -block, and more information can be found in the -[terraform docs][network-interface-tf]. - -> **_NOTE:_** When supplying the `network_interfaces` variable, networks -> associated with the `vm-instance` via use will be ignored in favor of the -> networks added in `network_interfaces`. In addition, `bandwidth_tier` and -> `disable_public_ips` will not apply to networks defined in -> `network_interfaces`. - -[network-interface-tf]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface - -### SSH key metadata - -This module will ignore all changes to the `ssh-keys` metadata field that are -typically set by [external Google Cloud tools that automate SSH access][gcpssh] -when not using OS Login. For example, clicking on the Google Cloud Console SSH -button next to VMs in the VM Instances list will temporarily modify VM metadata -to include a dynamically-generated SSH public key. - -[gcpssh]: https://cloud.google.com/compute/docs/connect/add-ssh-keys#metadata - -### Placement - -The `placement_policy` variable can be used to control where your VM instances -are physically located relative to each other within a zone. See the official -placement [guide][guide-link] and [api][api-link] documentation. - -[guide-link]: https://cloud.google.com/compute/docs/instances/define-instance-placement -[api-link]: https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement - -Use the following settings for compact placement: - -```yaml - ... - settings: - instance_count: 4 - machine_type: c2-standard-60 - placement_policy: - collocation: "COLLOCATED" -``` - -By default the above placement policy will always result in the most compact set -of VMs available. If you would like that provisioning failed if some level of -compactness is not obtainable, you can enforce this with the [`max_distance` -setting](https://cloud.google.com/compute/docs/instances/use-compact-placement-policies): - -```yaml - ... - settings: - instance_count: 4 - machine_type: c2-standard-60 - placement_policy: - collocation: "COLLOCATED" - max_distance: 1 -``` - -Use the following settings for spread placement: - -```yaml - ... - settings: - instance_count: 4 - machine_type: n2-standard-4 - placement_policy: - availability_domain_count: 2 -``` - -When `vm_count` is not set, as shown in the examples above, then the VMs will be -added to the placement policy incrementally. This is the **recommended way** to -use placement policies. - -If `vm_count` is specified then VMs will stay in pending state until the -specified number of VMs are created. See the warning below if using this field. - -> [!WARNING] -> When creating a compact placement using `vm_count` with more than 10 VMs, you -> must add `-parallelism=` argument on apply. For example if you have 15 VMs -> in a placement group: `terraform apply -parallelism=15`. This is because -> terraform self limits to 10 parallel requests by default but the create -> instance requests will not succeed until all VMs in the placement group have -> been requested, forming a deadlock. - -### GPU Support - -More information on GPU support in `vm-instance` and other Cluster Toolkit modules -can be found at [docs/gpu-support.md](../../../docs/gpu-support.md) - -## Lifecycle - -The `vm-instance` module will be replaced when the `instance_image` variable is -changed and `terraform apply` is run on the deployment group folder or -`gcluster deploy` is run. However, it will not be automatically replaced if a new -image is created in a family. - -To selectively replace the vm-instance(s), consider running terraform -`apply -replace` such as: - -> See https://developer.hashicorp.com/terraform/cli/commands/plan#replace-address for precise syntax terraform apply -replace=ADDRESS - -```shell -terraform state list -# search for the module ID and resource -terraform apply -replace="address" -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [google](#requirement\_google) | >= 4.73.0 | -| [google-beta](#requirement\_google-beta) | >= 6.13.0 | -| [null](#requirement\_null) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.73.0 | -| [google-beta](#provider\_google-beta) | >= 6.13.0 | -| [null](#provider\_null) | >= 3.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [gpu](#module\_gpu) | ../../internal/gpu-definition | n/a | -| [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | ../../scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_instance.compute_vm](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance) | resource | -| [google-beta_google_compute_resource_policy.placement_policy](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_resource_policy) | resource | -| [google_compute_address.compute_ip](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | -| [google_compute_disk.additional_disks](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [google_compute_disk.boot_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | -| [null_resource.image](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [null_resource.replace_vm_trigger_from_placement](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [add\_deployment\_name\_before\_prefix](#input\_add\_deployment\_name\_before\_prefix) | If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments.
See `name_prefix` for further details on resource naming behavior. | `bool` | `false` | no | -| [additional\_persistent\_disks](#input\_additional\_persistent\_disks) | Configurations of additional disks to be included on the partition nodes. | 
object({
    count = optional(number, 0)
    type  = optional(string, "pd-balanced")
    size  = optional(number, 200)
  })
| `{}` | no | -| [allocate\_ip](#input\_allocate\_ip) | If not null, allocate IPs with the given configuration. See details at
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address | 
object({
    address_type = optional(string, "INTERNAL")
    purpose      = optional(string),
    network_tier = optional(string),
    ip_version   = optional(string, "IPV4"),
  })
| `null` | no | -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [auto\_delete\_boot\_disk](#input\_auto\_delete\_boot\_disk) | Controls if boot disk should be auto-deleted when instance is deleted. | `bool` | `true` | no | -| [automatic\_restart](#input\_automatic\_restart) | Specifies if the instance should be restarted if it was terminated by Compute Engine (not a user). | `bool` | `null` | no | -| [bandwidth\_tier](#input\_bandwidth\_tier) | Tier 1 bandwidth increases the maximum egress bandwidth for VMs.
Using the `tier_1_enabled` setting will enable both gVNIC and TIER\_1 higher bandwidth networking.
Using the `gvnic_enabled` setting will only enable gVNIC and will not enable TIER\_1.
Note that TIER\_1 only works with specific machine families & shapes and must be using an image that supports gVNIC. See [official docs](https://cloud.google.com/compute/docs/networking/configure-vm-with-high-bandwidth-configuration) for more details. | `string` | `"not_enabled"` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment, will optionally be used name resources according to `name_prefix` | `string` | n/a | yes | -| [disable\_public\_ips](#input\_disable\_public\_ips) | If set to true, instances will not have public IPs | `bool` | `false` | no | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Size of disk for instances. | `number` | `200` | no | -| [disk\_type](#input\_disk\_type) | Disk type for instances. | `string` | `"pd-standard"` | no | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `[]` | no | -| [instance\_count](#input\_instance\_count) | Number of instances | `number` | `1` | no | -| [instance\_image](#input\_instance\_image) | Instance Image | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | -| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_ssd\_count](#input\_local\_ssd\_count) | The number of local SSDs to attach to each VM. See https://cloud.google.com/compute/docs/disks/local-ssd. | `number` | `0` | no | -| [local\_ssd\_interface](#input\_local\_ssd\_interface) | Interface to be used with local SSDs. Can be either 'NVME' or 'SCSI'. No effect unless `local_ssd_count` is also set. | `string` | `"NVME"` | no | -| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation | `string` | `"c2-standard-60"` | no | -| [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | -| [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | -| [name\_prefix](#input\_name\_prefix) | An optional name for all VM and disk resources.
If not supplied, `deployment_name` will be used.
When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set,
then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". | `string` | `null` | no | -| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces. The options match that of the terraform
network\_interface block of google\_compute\_instance. For descriptions of the
subfields or more information see the documentation:
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface

**\_NOTE:\_** If `network_interfaces` are set, `network_self_link` and
`subnetwork_self_link` will be ignored, even if they are provided through
the `use` field. `bandwidth_tier` and `disable_public_ips` also do not apply
to network interfaces defined in this variable.

Subfields:
network (string, required if subnetwork is not supplied)
subnetwork (string, required if network is not supplied)
subnetwork\_project (string, optional)
network\_ip (string, optional)
nic\_type (string, optional, choose from ["GVNIC", "VIRTIO\_NET", "MRDMA", "IRDMA"])
stack\_type (string, optional, choose from ["IPV4\_ONLY", "IPV4\_IPV6"])
queue\_count (number, optional)
access\_config (object, optional)
ipv6\_access\_config (object, optional)
alias\_ip\_range (list(object), optional) | 
list(object({
    network            = string,
    subnetwork         = string,
    subnetwork_project = string,
    network_ip         = string,
    nic_type           = string,
    stack_type         = string,
    queue_count        = number,
    access_config = list(object({
      nat_ip                 = string,
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    ipv6_access_config = list(object({
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    alias_ip_range = list(object({
      ip_cidr_range         = string,
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the VM. Can use "default" for the default network. | `string` | `null` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE` | `string` | `null` | no | -| [placement\_policy](#input\_placement\_policy) | Control where your VM instances are physically located relative to each other within a zone.
See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_resource_policy#nested_group_placement_policy | `any` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [provisioning\_model](#input\_provisioning\_model) | Provisioning model for cloud instance. | `string` | `null` | no | -| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | -| [reservation\_name](#input\_reservation\_name) | Name of the reservation to use for VM resources, should be in one of the following formats:
- projects/PROJECT\_ID/reservations/RESERVATION\_NAME
- RESERVATION\_NAME

Must be a "SPECIFIC\_RESERVATION"
Set to empty string if using no reservation or automatically-consumed reservations | `string` | `""` | no | -| [service\_account](#input\_service\_account) | DEPRECATED - Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the node pool | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [spot](#input\_spot) | DEPRECATED - Use `provisioning_model` instead. | `bool` | `null` | no | -| [startup\_script](#input\_startup\_script) | Startup script used on the instance | `string` | `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to attach the VM. | `string` | `null` | no | -| [tags](#input\_tags) | Network tags, provided as a list | `list(string)` | `[]` | no | -| [threads\_per\_core](#input\_threads\_per\_core) | Sets the number of threads per physical core. By setting threads\_per\_core
to 2, Simultaneous Multithreading (SMT) is enabled extending the total number
of virtual cores. For example, a machine of type c2-standard-60 will have 60
virtual cores with threads\_per\_core equal to 2. With threads\_per\_core equal
to 1 (SMT turned off), only the 30 physical cores will be available on the VM.

The default value of \"0\" will turn off SMT for supported machine types, and
will fall back to GCE defaults for unsupported machine types (t2d, shared-core
instances, or instances with less than 2 vCPU).

Disabling SMT can be more performant in many HPC workloads, therefore it is
disabled by default where compatible.

null = SMT configuration will use the GCE defaults for the machine type
0 = SMT will be disabled where compatible (default)
1 = SMT will always be disabled (will fail on incompatible machine types)
2 = SMT will always be enabled (will fail on incompatible machine types) | `number` | `0` | no | -| [zone](#input\_zone) | Compute Platform zone | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [external\_ip](#output\_external\_ip) | External IP of the instances (if enabled) | -| [instructions](#output\_instructions) | Instructions on how to SSH into the created VM. Commands may fail depending on VM configuration and IAM permissions. | -| [internal\_ip](#output\_internal\_ip) | Internal IP of the instances | -| [name](#output\_name) | Names of instances created | -| [self\_link](#output\_self\_link) | The tuple URIs of the created instances | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/compute_image.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/compute_image.tf deleted file mode 100644 index 7a7fe02307..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/compute_image.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_compute_image" "compute_image" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = try(var.instance_image.project, null) - - lifecycle { - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/main.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/main.tf deleted file mode 100644 index 9618ea9179..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/main.tf +++ /dev/null @@ -1,349 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "vm-instance", ghpc_role = "compute" }) -} - -module "gpu" { - source = "../../internal/gpu-definition" - - machine_type = var.machine_type - guest_accelerator = var.guest_accelerator -} - -locals { - guest_accelerator = module.gpu.guest_accelerator - - native_fstype = [] - startup_script = local.startup_from_network_storage != null ? ( - { startup-script = local.startup_from_network_storage }) : {} - network_storage = var.network_storage != null ? ( - { network_storage = jsonencode(var.network_storage) }) : {} - - prefix_optional_deployment_name = var.name_prefix != null ? var.name_prefix : var.deployment_name - prefix_always_deployment_name = var.name_prefix != null ? "${var.deployment_name}-${var.name_prefix}" : var.deployment_name - resource_prefix = var.add_deployment_name_before_prefix ? local.prefix_always_deployment_name : local.prefix_optional_deployment_name - - enable_gvnic = var.bandwidth_tier != "not_enabled" - enable_tier_1 = var.bandwidth_tier == "tier_1_enabled" - - provisioning_model = var.provisioning_model - - spot = var.provisioning_model == "SPOT" - - # compact_placement : true when placement policy is provided and collocation set; false if unset - compact_placement = try(var.placement_policy.collocation, null) != null - - gpu_attached = contains(["a2", "g2"], local.machine_family) || length(local.guest_accelerator) > 0 - - # both of these must be false if either compact placement or preemptible/spot instances are used - # automatic restart is tolerant of GPUs while on host maintenance is not - automatic_restart_default = local.compact_placement || local.spot ? false : null - on_host_maintenance_default = local.compact_placement || local.spot || local.gpu_attached ? "TERMINATE" : "MIGRATE" - - automatic_restart = ( - var.automatic_restart != null - ? var.automatic_restart - : local.automatic_restart_default - ) - - on_host_maintenance = ( - var.on_host_maintenance != null - ? var.on_host_maintenance - : local.on_host_maintenance_default - ) - - oslogin_api_values = { - "DISABLE" = "FALSE" - "ENABLE" = "TRUE" - } - enable_oslogin = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } - - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - # Network Interfaces - # Support for `use` input and base network parameters like `network_self_link` and `subnetwork_self_link` - empty_access_config = { - nat_ip = null, - public_ptr_domain_name = null, - network_tier = null - } - default_network_interface = { - network = var.network_self_link - subnetwork = var.subnetwork_self_link - subnetwork_project = null # will populate from subnetwork_self_link - network_ip = null - nic_type = local.enable_gvnic ? "GVNIC" : null - stack_type = null - queue_count = null - access_config = var.disable_public_ips ? [] : [local.empty_access_config] - ipv6_access_config = [] - alias_ip_range = [] - } - network_interfaces = coalescelist(var.network_interfaces, [local.default_network_interface]) - network_interfaces_with_ips = var.allocate_ip == null ? local.network_interfaces : [ - for i, interface in local.network_interfaces : - merge(interface, { - network_ip = google_compute_address.compute_ip[i].address - }) - ] -} - -resource "null_resource" "image" { - triggers = { - name = try(var.instance_image.name, null), - family = try(var.instance_image.family, null), - project = try(var.instance_image.project, null) - } -} - -resource "google_compute_disk" "boot_disk" { - project = var.project_id - - count = var.instance_count - - name = "${local.resource_prefix}-boot-disk-${count.index}" - image = data.google_compute_image.compute_image.self_link - type = var.disk_type - size = var.disk_size_gb - labels = local.labels - zone = var.zone - - lifecycle { - replace_triggered_by = [null_resource.image] - - ignore_changes = [ - image - ] - } -} - -resource "google_compute_disk" "additional_disks" { - project = var.project_id - - count = var.instance_count * var.additional_persistent_disks.count - - # NB: this resource array must be sliced accounting for var.instance_count - name = "${local.resource_prefix}-disk-${count.index}" - type = var.additional_persistent_disks.type - size = var.additional_persistent_disks.size - labels = local.labels - zone = var.zone -} - -resource "google_compute_resource_policy" "placement_policy" { - project = var.project_id - provider = google-beta - - count = var.placement_policy != null ? 1 : 0 - name = "${local.resource_prefix}-vm-instance-placement" - group_placement_policy { - vm_count = try(var.placement_policy.vm_count, null) - availability_domain_count = try(var.placement_policy.availability_domain_count, null) - collocation = try(var.placement_policy.collocation, null) - max_distance = try(var.placement_policy.max_distance, null) - } -} - -resource "null_resource" "replace_vm_trigger_from_placement" { - triggers = { - vm_count = try(tostring(var.placement_policy.vm_count), "") - availability_domain_count = try(tostring(var.placement_policy.availability_domain_count), "") - max_distance = try(tostring(var.placement_policy.max_distance), "") - collocation = try(var.placement_policy.collocation, "") - } -} - -resource "google_compute_address" "compute_ip" { - project = var.project_id - - count = var.allocate_ip != null ? length(local.network_interfaces) : 0 - - name = "${local.resource_prefix}-${count.index}" - - address = local.network_interfaces[count.index].network_ip - region = var.region - network = can(coalesce(local.network_interfaces[count.index].subnetwork)) ? null : local.network_interfaces[count.index].network - subnetwork = local.network_interfaces[count.index].subnetwork - address_type = var.allocate_ip.address_type - purpose = var.allocate_ip.purpose - network_tier = var.allocate_ip.network_tier - ip_version = var.allocate_ip.ip_version -} - -resource "google_compute_instance" "compute_vm" { - project = var.project_id - provider = google-beta - - count = var.instance_count - - depends_on = [var.network_self_link, var.network_storage] - - name = "${local.resource_prefix}-${count.index}" - min_cpu_platform = var.min_cpu_platform - machine_type = var.machine_type - zone = var.zone - - resource_policies = google_compute_resource_policy.placement_policy[*].self_link - - tags = var.tags - labels = local.labels - - boot_disk { - source = google_compute_disk.boot_disk[count.index].self_link - device_name = google_compute_disk.boot_disk[count.index].name - auto_delete = var.auto_delete_boot_disk - } - - dynamic "attached_disk" { - for_each = slice( - google_compute_disk.additional_disks, - var.additional_persistent_disks.count * count.index, - var.additional_persistent_disks.count * count.index + var.additional_persistent_disks.count, - ) - - content { - source = attached_disk.value.self_link - device_name = "additional-disk-${attached_disk.key}" - mode = "READ_WRITE" - } - } - - dynamic "scratch_disk" { - for_each = range(var.local_ssd_count) - content { - interface = var.local_ssd_interface - } - } - - dynamic "network_interface" { - for_each = local.network_interfaces_with_ips - - content { - network = network_interface.value.network - subnetwork = network_interface.value.subnetwork - subnetwork_project = network_interface.value.subnetwork_project - network_ip = network_interface.value.network_ip - nic_type = network_interface.value.nic_type - stack_type = network_interface.value.stack_type - queue_count = network_interface.value.queue_count - dynamic "access_config" { - for_each = network_interface.value.access_config - content { - nat_ip = access_config.value.nat_ip - public_ptr_domain_name = access_config.value.public_ptr_domain_name - network_tier = access_config.value.network_tier - } - } - dynamic "ipv6_access_config" { - for_each = network_interface.value.ipv6_access_config - content { - public_ptr_domain_name = ipv6_access_config.value.public_ptr_domain_name - network_tier = ipv6_access_config.value.network_tier - } - } - dynamic "alias_ip_range" { - for_each = network_interface.value.alias_ip_range - content { - ip_cidr_range = alias_ip_range.value.ip_cidr_range - subnetwork_range_name = alias_ip_range.value.subnetwork_range_name - } - } - } - } - - network_performance_config { - total_egress_bandwidth_tier = local.enable_tier_1 ? "TIER_1" : "DEFAULT" - } - - service_account { - email = var.service_account_email - scopes = var.service_account_scopes - } - - dynamic "guest_accelerator" { - for_each = local.guest_accelerator - content { - count = guest_accelerator.value.count - type = guest_accelerator.value.type - } - } - - scheduling { - on_host_maintenance = local.on_host_maintenance - automatic_restart = local.automatic_restart - preemptible = local.spot - provisioning_model = local.provisioning_model - } - - dynamic "advanced_machine_features" { - for_each = local.set_threads_per_core ? [1] : [] - content { - threads_per_core = local.threads_per_core # relies on threads_per_core_calc.tf - } - } - - dynamic "reservation_affinity" { - for_each = var.reservation_name == "" ? [] : [1] - content { - type = "SPECIFIC_RESERVATION" - specific_reservation { - key = "compute.googleapis.com/reservation-name" - values = [var.reservation_name] - } - } - } - - metadata = merge( - local.network_storage, - local.startup_script, - local.enable_oslogin, - local.disable_automatic_updates_metadata, - var.metadata - ) - - lifecycle { - ignore_changes = [ - metadata["ssh-keys"], - ] - - replace_triggered_by = [ - null_resource.replace_vm_trigger_from_placement - ] - - precondition { - condition = (length(var.network_interfaces) == 0) != (var.network_self_link == null && var.subnetwork_self_link == null) - error_message = "Exactly one of network_interfaces or network_self_link/subnetwork_self_link must be specified." - } - precondition { - condition = alltrue([for interface in var.network_interfaces : interface.network_ip == null]) || var.instance_count == 1 - error_message = <<-EOT - The network_ip cannot be statically set on vm-instance when the VM instance_count is greater than 1. - Either set the network_ip to null to allow it to be set dynamically for all instances, or create modules for each VM instance with its own network interface. - EOT - } - precondition { - condition = !contains([ - "c3-:pd-standard", - "h3-:pd-standard", - "h3-:pd-ssd", - ], "${substr(var.machine_type, 0, 3)}:${var.disk_type}") - error_message = "A disk_type=${var.disk_type} cannot be used with machine_type=${var.machine_type}." - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/outputs.tf deleted file mode 100644 index eab8cb56bd..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/outputs.tf +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "name" { - description = "Names of instances created" - value = google_compute_instance.compute_vm[*].name -} - -output "self_link" { - description = "The tuple URIs of the created instances" - value = google_compute_instance.compute_vm[*].self_link -} - -output "external_ip" { - description = "External IP of the instances (if enabled)" - value = try(google_compute_instance.compute_vm[*].network_interface[0].access_config[0].nat_ip, []) -} - -output "internal_ip" { - description = "Internal IP of the instances" - value = google_compute_instance.compute_vm[*].network_interface[0].network_ip -} - -locals { - first_instance_link = try(google_compute_instance.compute_vm[0].self_link, "no-instance") - ssh_instructions = <<-EOT - Use the following commands to SSH into the first VM created: - gcloud compute ssh ${local.first_instance_link} --project ${var.project_id} - If not accessible from the public internet, use an SSH tunnel through IAP: - gcloud compute ssh ${local.first_instance_link} --tunnel-through-iap --project ${var.project_id} - EOT -} - -output "instructions" { - description = "Instructions on how to SSH into the created VM. Commands may fail depending on VM configuration and IAM permissions." - value = var.instance_count > 0 ? local.ssh_instructions : "No instances were created." -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf deleted file mode 100644 index 02bc58e4f7..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/startup_from_network_storage.tf +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# This file is meant to be reused by multiple modules. -# "inputs": -# local.native_fstype : list of file systems that are supported automatically, but looking at the metadata. -# var.network_storage : to be passed into metadata somewhere else (not here) -# var.startup_script : to be changed into a more complete file system with all the fs runners - -# "outputs": -# local.startup_from_network_storage : A full startup script with all the runners that are not supported -# natively and were included in the network_storage structure - -locals { - startup_script_network_storage = [ - for ns in var.network_storage : - ns if !contains(local.native_fstype, ns.fs_type) - ] - # Pull out runners to include in startup script - storage_client_install_runners = [ - for ns in local.startup_script_network_storage : - ns.client_install_runner if ns.client_install_runner != null - ] - mount_runners = [ - for ns in local.startup_script_network_storage : - ns.mount_runner if ns.mount_runner != null - ] - - startup_script_runner = [{ - content = var.startup_script != null ? var.startup_script : "echo 'No user provided startup script.'" - destination = "passed_startup_script.sh" - type = "shell" - }] - - full_runner_list = concat( - local.storage_client_install_runners, - local.mount_runners, - local.startup_script_runner - ) - - startup_from_network_storage = module.netstorage_startup_script.startup_script -} - -module "netstorage_startup_script" { - source = "../../scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.full_runner_list -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf deleted file mode 100644 index e582db33da..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/threads_per_core_calc.tf +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# This file is meant to be reused by multiple modules. -# "description": Allows for 'threads_per_core=0: SMT will be disabled where compatible (default)' - -# "inputs": -# var.machine_type: Machine type for the instance being evaluated. -# var.threads_per_core : Sets the number of threads per physical core, where 0 -# has behavior described in description. - -# "outputs": -# local.set_threads_per_core: bool that tells if threads per core should be set, -# to be used with a dynamic block. -# local.threads_per_core: actual threads_per_core to be used. - -locals { - machine_vals = split("-", var.machine_type) - machine_family = local.machine_vals[0] - machine_shared_core = length(local.machine_vals) <= 2 - machine_vcpus = try(parseint(local.machine_vals[2], 10), 1) - - smt_capable_family = !contains(["t2d", "t2a"], local.machine_family) - smt_capable_vcpu = local.machine_vcpus >= 2 - - smt_capable = local.smt_capable_family && local.smt_capable_vcpu && !local.machine_shared_core - set_threads_per_core = var.threads_per_core != null && (var.threads_per_core == 0 && local.smt_capable || try(var.threads_per_core >= 1, false)) - threads_per_core = var.threads_per_core == 2 ? 2 : 1 -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/variables.tf deleted file mode 100644 index 5519b8cd40..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/variables.tf +++ /dev/null @@ -1,452 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "instance_count" { - description = "Number of instances" - type = number - default = 1 -} - -variable "instance_image" { - description = "Instance Image" - type = map(string) - default = { - project = "cloud-hpc-image-public" - family = "hpc-rocky-linux-8" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "disk_size_gb" { - description = "Size of disk for instances." - type = number - default = 200 -} - -variable "disk_type" { - description = "Disk type for instances." - type = string - default = "pd-standard" -} - -variable "auto_delete_boot_disk" { - description = "Controls if boot disk should be auto-deleted when instance is deleted." - type = bool - default = true -} - -variable "local_ssd_count" { - description = "The number of local SSDs to attach to each VM. See https://cloud.google.com/compute/docs/disks/local-ssd." - type = number - default = 0 -} - -variable "local_ssd_interface" { - description = "Interface to be used with local SSDs. Can be either 'NVME' or 'SCSI'. No effect unless `local_ssd_count` is also set." - type = string - default = "NVME" -} - -variable "additional_persistent_disks" { - description = "Configurations of additional disks to be included on the partition nodes." - type = object({ - count = optional(number, 0) - type = optional(string, "pd-balanced") - size = optional(number, 200) - }) - default = {} -} - -variable "name_prefix" { - description = <<-EOT - An optional name for all VM and disk resources. - If not supplied, `deployment_name` will be used. - When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set, - then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". - EOT - type = string - default = null -} - -variable "add_deployment_name_before_prefix" { - description = <<-EOT - If true, the names of VMs and disks will always be prefixed with `deployment_name` to enable uniqueness across deployments. - See `name_prefix` for further details on resource naming behavior. - EOT - type = bool - default = false -} - -variable "disable_public_ips" { - description = "If set to true, instances will not have public IPs" - type = bool - default = false -} - -variable "machine_type" { - description = "Machine type to use for the instance creation" - type = string - default = "c2-standard-60" -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured." - type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "deployment_name" { - description = "Name of the deployment, will optionally be used name resources according to `name_prefix`" - type = string -} - -variable "labels" { - description = "Labels to add to the instances. Key-value pairs." - type = map(string) -} - -variable "service_account_email" { - description = "Service account e-mail address to use with the node pool" - type = string - default = null -} - -variable "service_account_scopes" { - description = "Scopes to to use with the node pool." - type = set(string) - default = ["https://www.googleapis.com/auth/cloud-platform"] -} - -# tflint-ignore: terraform_unused_declarations -variable "service_account" { - description = "DEPRECATED - Use `service_account_email` and `service_account_scopes` instead." - type = object({ - email = string, - scopes = set(string) - }) - default = null - validation { - condition = var.service_account == null - error_message = "The 'service_account' setting is deprecated, please use 'var.service_account_email' and 'var.service_account_scopes' instead." - } -} - -variable "network_self_link" { - description = "The self link of the network to attach the VM. Can use \"default\" for the default network." - type = string - default = null -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork to attach the VM." - type = string - default = null -} - -variable "network_interfaces" { - description = <<-EOT - A list of network interfaces. The options match that of the terraform - network_interface block of google_compute_instance. For descriptions of the - subfields or more information see the documentation: - https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface - - **_NOTE:_** If `network_interfaces` are set, `network_self_link` and - `subnetwork_self_link` will be ignored, even if they are provided through - the `use` field. `bandwidth_tier` and `disable_public_ips` also do not apply - to network interfaces defined in this variable. - - Subfields: - network (string, required if subnetwork is not supplied) - subnetwork (string, required if network is not supplied) - subnetwork_project (string, optional) - network_ip (string, optional) - nic_type (string, optional, choose from ["GVNIC", "VIRTIO_NET", "MRDMA", "IRDMA"]) - stack_type (string, optional, choose from ["IPV4_ONLY", "IPV4_IPV6"]) - queue_count (number, optional) - access_config (object, optional) - ipv6_access_config (object, optional) - alias_ip_range (list(object), optional) - EOT - type = list(object({ - network = string, - subnetwork = string, - subnetwork_project = string, - network_ip = string, - nic_type = string, - stack_type = string, - queue_count = number, - access_config = list(object({ - nat_ip = string, - public_ptr_domain_name = string, - network_tier = string - })), - ipv6_access_config = list(object({ - public_ptr_domain_name = string, - network_tier = string - })), - alias_ip_range = list(object({ - ip_cidr_range = string, - subnetwork_range_name = string - })) - })) - default = [] - validation { - condition = alltrue([ - for ni in var.network_interfaces : (ni.network == null) != (ni.subnetwork == null) - ]) - error_message = "All additional network interfaces must define exactly one of \"network\" or \"subnetwork\"." - } - validation { - condition = alltrue([ - for ni in var.network_interfaces : ni.nic_type == "GVNIC" || ni.nic_type == "VIRTIO_NET" || ni.nic_type == "MRDMA" || ni.nic_type == "IRDMA" || ni.nic_type == null - ]) - error_message = "In the variable network_interfaces, field \"nic_type\" must be \"GVNIC\", \"VIRTIO_NET\", \"MRDMA\", \"IRDMA\", or null." - } - validation { - condition = alltrue([ - for ni in var.network_interfaces : ni.stack_type == "IPV4_ONLY" || ni.stack_type == "IPV4_IPV6" || ni.stack_type == null - ]) - error_message = "In the variable network_interfaces, field \"stack_type\" must be either \"IPV4_ONLY\", \"IPV4_IPV6\" or null." - } -} - -variable "region" { - description = "The region to deploy to" - type = string -} - -variable "zone" { - description = "Compute Platform zone" - type = string -} - -variable "metadata" { - description = "Metadata, provided as a map" - type = map(string) - default = {} -} - -variable "startup_script" { - description = "Startup script used on the instance" - type = string - default = null -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = string, - count = number - })) - default = [] - nullable = false -} - -variable "automatic_restart" { - description = "Specifies if the instance should be restarted if it was terminated by Compute Engine (not a user)." - type = bool - default = null -} - -variable "on_host_maintenance" { - description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE`" - type = string - default = null - validation { - condition = var.on_host_maintenance == null ? true : contains(["MIGRATE", "TERMINATE"], var.on_host_maintenance) - error_message = "When set, the on_host_maintenance must be set to MIGRATE or TERMINATE." - } -} - -variable "bandwidth_tier" { - description = <= 0, false) && try(var.threads_per_core <= 2, false) - error_message = "Allowed values for threads_per_core are \"null\", \"0\", \"1\", \"2\"." - } - -} - -variable "enable_oslogin" { - description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." - type = string - default = "ENABLE" - validation { - condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) - error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." - } -} - -variable "allocate_ip" { - description = <<-EOT - If not null, allocate IPs with the given configuration. See details at - https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address - EOT - type = object({ - address_type = optional(string, "INTERNAL") - purpose = optional(string), - network_tier = optional(string), - ip_version = optional(string, "IPV4"), - }) - default = null -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} - -variable "reservation_name" { - description = <<-EOD - Name of the reservation to use for VM resources, should be in one of the following formats: - - projects/PROJECT_ID/reservations/RESERVATION_NAME - - RESERVATION_NAME - - Must be a "SPECIFIC_RESERVATION" - Set to empty string if using no reservation or automatically-consumed reservations - EOD - type = string - default = "" - nullable = false - - validation { - condition = length(regexall("^((projects/([a-z0-9-]+)/reservations/)?([a-z0-9-]+))?$", var.reservation_name)) > 0 - error_message = "Reservation name must be either empty or in the format '[projects/PROJECT_ID/reservations/]RESERVATION_NAME', [...] is an optional part." - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/versions.tf deleted file mode 100644 index 4b2786e2af..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/compute/vm-instance/versions.tf +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.73.0" - } - - google-beta = { - source = "hashicorp/google-beta" - version = ">= 6.13.0" - } - null = { - source = "hashicorp/null" - version = ">= 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.57.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.57.0" - } - - required_version = ">= 1.3.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/README.md b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/README.md deleted file mode 100644 index 0ee90e1d63..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/README.md +++ /dev/null @@ -1,248 +0,0 @@ -## Description - -This module creates a [filestore](https://cloud.google.com/filestore) -instance. Filestore is a high performance network file system that can be -mounted to one or more compute VMs. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). - -### Deletion protection - -We recommend considering enabling [Filestore deletion protection][fdp]. Deletion -protection will prevent unintentional deletion of an entire Filestore instance. -It does not prevent deletion of files within the Filestore instance when mounted -by a VM. It is not available on some [tiers](#filestore-tiers), including the -default BASIC\_HDD tier or BASIC\_SSD tier. Follow the documentation link for -up to date details. - -Usage can be enabled in a blueprint with, for example: - -```yaml - - id: homefs - source: modules/file-system/filestore - use: [network] - settings: - deletion_protection: - enabled: true - reason: Avoid data loss - filestore_tier: ZONAL - local_mount: /home - size_gb: 1024 -``` - -[fdp]: https://cloud.google.com/filestore/docs/deletion-protection - -### Filestore tiers - -At the time of writing, Filestore supports 5 [tiers of service][tiers] that are -specified in the Toolkit using the following names: - -- Basic HDD: "BASIC\_HDD" ([preferred][tierapi]) or "STANDARD" (deprecated) -- Basic SSD: "BASIC\_SSD" ([preferred][tierapi]) or "PREMIUM" (deprecated) -- Zonal: "ZONAL" -- Enterprise: "ENTERPRISE" -- Regional: "REGIONAL" - -[tierapi]: https://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier - -**Please review the minimum storage requirements for each tier**. The Terraform -module can only enforce the minimum value of the `size_gb` parameter for the -lowest tier of service. If you supply a value that is too low, Filestore -creation will fail when you run `terraform apply`. - -[tiers]: https://cloud.google.com/filestore/docs/service-tiers - -### Filestore protocols and mount options -After Filestore instance is created, you can mount this to the compute node -using different mount options. Toolkit uses [default mount options](https://linux.die.net/man/8/mount) -for all tier services. Filestore has recommended mount options for different -service tiers which may overall improve performance. These can be found here: -[recommended mount options.](https://cloud.google.com/filestore/docs/mounting-fileshares) -While creating filestore module, you can overwrite these mount options as -mentioned below. - -```yaml -- id: homefs - source: modules/file-system/filestore - use: [network1] - settings: - local_mount: /homefs - mount_options: defaults,hard,timeo=600,retrans=3,_netdev -``` - -Filestore supports NFS protocols `NFS_V3` (default) and `NFS_V4_1`. Protocol support depends on the selected tier: -- `NFS_V3`: Supported on all tiers (`BASIC_HDD`, `BASIC_SSD`, `HIGH_SCALE_SSD`, `ZONAL`, `ENTERPRISE`). -- `NFS_V4_1`: Supported only on `HIGH_SCALE_SSD`, `ZONAL`, `REGIONAL`, and `ENTERPRISE`. -This can be specified at creation time via the `protocol` variable. By default, `NFS_V3` is used for compatibility. -See the example below and [this page](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance#protocol-1) for more information. - -```yaml -- id: homefs - source: modules/file-system/filestore - use: [network1] - settings: - local_mount: /homefs - protocol: NFS_V4_1 - filestore_tier: ZONAL -``` - -### Filestore quota - -Your project must have unused quota for Cloud Filestore in the region you will -provision the storage. This can be found by browsing to the [Quota tab within IAM -& Admin](https://console.cloud.google.com/iam-admin/quotas) in the Cloud Console. -Please note that there are separate quota limits for HDD and SSD storage. - -All projects begin with 0 available quota for High Scale SSD tier. To use this -tier, [make a request and wait for it to be approved][hs-ssd-quota]. - -[hs-ssd-quota]: https://cloud.google.com/filestore/docs/high-scale - -### Example - Basic HDD - -The Filestore instance defined below will have the following attributes: - -- (default) `BASIC_HDD` tier -- (default) 1TiB capacity -- `homefs` module ID -- mount point at `/home` -- connected to the network defined in the `network1` module - -```yaml -- id: homefs - source: modules/file-system/filestore - use: [network1] - settings: - local_mount: /home -``` - -### Example - High Scale SSD - -The Filestore instance defined below will have the following attributes: - -- `HIGH_SCALE_SSD` tier -- 10TiB capacity -- `highscale` module ID -- mount point at `/projects` -- connected to the VPC network defined in the `network1` module - -```yaml -- id: highscale - source: modules/file-system/filestore - use: [network1] - settings: - filestore_tier: HIGH_SCALE_SSD - size_gb: 10240 - local_mount: /projects -``` - -## Mounting - -To mount the Filestore instance you must first ensure that the NFS client has -been installed and then call the proper `mount` command. - -Both of these steps are automatically handled with the use of the `use` command -in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in -the network storage doc for a complete list of supported modules. -See the [hpc-slurm](../../../examples/hpc-slurm.yaml) for -an example of using this module with Slurm. - -If mounting is not automatically handled as described above, the `filestore` -module outputs runners that can be used with the startup-script module to -install the client and mount the file system. See the following example: - -```yaml - - id: filestore - source: modules/file-system/filestore - use: [network1] - settings: {local_mount: /scratch} - - - id: mount-at-startup - source: modules/scripts/startup-script - settings: - runners: - - $(filestore.install_nfs_client_runner) - - $(filestore.mount_runner) - -``` - -[matrix]: ../../../docs/network_storage.md#compatibility-matrix - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [google](#requirement\_google) | >= 6.4 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.4 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_filestore_instance.filestore_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [connect\_mode](#input\_connect\_mode) | Used to select mode - supported values DIRECT\_PEERING and PRIVATE\_SERVICE\_ACCESS. | `string` | `"DIRECT_PEERING"` | no | -| [deletion\_protection](#input\_deletion\_protection) | Configure Filestore instance deletion protection | 
object({
    enabled = optional(bool, false)
    reason  = optional(string)
  })
| 
{
  "enabled": false
}
| no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the filestore instance if no name is specified. | `string` | n/a | yes | -| [description](#input\_description) | A description of the filestore instance. | `string` | `""` | no | -| [filestore\_share\_name](#input\_filestore\_share\_name) | Name of the file system share on the instance. | `string` | `"nfsshare"` | no | -| [filestore\_tier](#input\_filestore\_tier) | The service tier of the instance. | `string` | `"BASIC_HDD"` | no | -| [labels](#input\_labels) | Labels to add to the filestore instance. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_mount](#input\_local\_mount) | Mountpoint for this filestore instance. Note: If set to the same as the `filestore_share_name`, it will trigger a known Slurm bug ([troubleshooting](../../../docs/slurm-troubleshooting.md)). | `string` | `"/shared"` | no | -| [mount\_options](#input\_mount\_options) | NFS mount options to mount file system. | `string` | `"defaults,_netdev"` | no | -| [name](#input\_name) | The resource name of the instance. | `string` | `null` | no | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | -| [nfs\_export\_options](#input\_nfs\_export\_options) | Define NFS export options. | 
list(object({
    access_mode = optional(string)
    ip_ranges   = optional(list(string))
    squash_mode = optional(string)
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | ID of project in which Filestore instance will be created. | `string` | n/a | yes | -| [protocol](#input\_protocol) | NFS protocol version. Default is NFS\_V3. NFS\_V4\_1 is only supported with HIGH\_SCALE\_SSD, ZONAL, REGIONAL, and ENTERPRISE tiers. | `string` | `"NFS_V3"` | no | -| [region](#input\_region) | Location for Filestore instances at Enterprise tier. | `string` | n/a | yes | -| [reserved\_ip\_range](#input\_reserved\_ip\_range) | Reserved IP range for Filestore instance. Users are encouraged to set to null
for automatic selection. If supplied, it must be:

CIDR format when var.connect\_mode == "DIRECT\_PEERING"
Named IP Range when var.connect\_mode == "PRIVATE\_SERVICE\_ACCESS"

See Cloud documentation for more details:

https://cloud.google.com/filestore/docs/creating-instances#configure_a_reserved_ip_address_range | `string` | `null` | no | -| [size\_gb](#input\_size\_gb) | Storage size of the filestore instance in GB. | `number` | `1024` | no | -| [zone](#input\_zone) | Location for Filestore instances below Enterprise tier. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [capacity\_gb](#output\_capacity\_gb) | File share capacity in GiB. | -| [filestore\_id](#output\_filestore\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}` | -| [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client | -| [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module | -| [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script
module will automatically handle installation of ansible.
- id: example-startup-script
source: modules/scripts/startup-script
settings:
runners:
- $(your-fs-id.mount\_runner)
... | -| [network\_storage](#output\_network\_storage) | Describes a filestore instance. | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/main.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/main.tf deleted file mode 100644 index ce035dbb2b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/main.tf +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "filestore", ghpc_role = "file-system" }) -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -locals { - is_high_capacity_tier = contains(["HIGH_SCALE_SSD", "ZONAL", "REGIONAL"], var.filestore_tier) && var.size_gb >= 10240 && var.size_gb <= 102400 - - timeouts = local.is_high_capacity_tier ? [1] : [] - server_ip = google_filestore_instance.filestore_instance.networks[0].ip_addresses[0] - remote_mount = format("/%s", google_filestore_instance.filestore_instance.file_shares[0].name) - fs_type = "nfs" - mount_options = var.mount_options - - install_nfs_client_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/install-nfs-client.sh" - "destination" = "install-nfs${replace(var.local_mount, "/", "_")}.sh" - } - mount_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/mount.sh" - "args" = "\"${local.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" - "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" - } - - # id format: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network#id - split_network_id = split("/", var.network_id) - network_name = local.split_network_id[4] - network_project = local.split_network_id[1] - shared_vpc = local.network_project != var.project_id -} - -resource "google_filestore_instance" "filestore_instance" { - project = var.project_id - - name = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" - description = var.description - location = contains(["ENTERPRISE", "REGIONAL"], var.filestore_tier) ? var.region : var.zone - tier = var.filestore_tier - protocol = var.protocol - - deletion_protection_enabled = var.deletion_protection.enabled - deletion_protection_reason = var.deletion_protection.reason - - file_shares { - capacity_gb = var.size_gb - name = var.filestore_share_name - dynamic "nfs_export_options" { - for_each = var.nfs_export_options - content { - access_mode = nfs_export_options.value.access_mode - ip_ranges = nfs_export_options.value.ip_ranges - squash_mode = nfs_export_options.value.squash_mode - } - } - } - - labels = local.labels - - networks { - network = local.shared_vpc ? var.network_id : local.network_name - connect_mode = var.connect_mode - modes = ["MODE_IPV4"] - reserved_ip_range = var.reserved_ip_range - } - - dynamic "timeouts" { - for_each = local.timeouts - content { - create = "1h" - update = "1h" - delete = "1h" - } - } - - lifecycle { - precondition { - condition = ( - var.reserved_ip_range == null || - var.connect_mode == "PRIVATE_SERVICE_ACCESS" || - var.connect_mode == "DIRECT_PEERING" && can(cidrhost(var.reserved_ip_range, 0)) && contains(["24", "29"], try(split("/", var.reserved_ip_range)[1], "")) - ) - error_message = <<-EOT - If connect_mode is set to DIRECT_PEERING and reserved_ip_range is - specified then it must be a CIDR IP range with suffix range size 29 for - BASIC_HDD or BASIC_SSD tiers. Otherwise the range size must be 24. - EOT - } - - precondition { - condition = !startswith(var.filestore_tier, "BASIC") || var.protocol != "NFS_V4_1" - error_message = "NFS_V4_1 is not supported on BASIC Filestore tiers." - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/metadata.yaml deleted file mode 100644 index 5298336f09..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - file.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/outputs.tf deleted file mode 100644 index bd9126798c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/outputs.tf +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "network_storage" { - description = "Describes a filestore instance." - value = { - server_ip = local.server_ip - remote_mount = local.remote_mount - local_mount = var.local_mount - fs_type = local.fs_type - mount_options = local.mount_options - client_install_runner = local.install_nfs_client_runner - mount_runner = local.mount_runner - } -} - -output "install_nfs_client" { - description = "Script for installing NFS client" - value = file("${path.module}/scripts/install-nfs-client.sh") -} - -output "install_nfs_client_runner" { - description = "Runner to install NFS client using the startup-script module" - value = local.install_nfs_client_runner -} - -output "mount_runner" { - description = <<-EOT - Runner to mount the file-system using an ansible playbook. The startup-script - module will automatically handle installation of ansible. - - id: example-startup-script - source: modules/scripts/startup-script - settings: - runners: - - $(your-fs-id.mount_runner) - ... - EOT - value = local.mount_runner -} - -output "filestore_id" { - description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}`" - value = google_filestore_instance.filestore_instance.id -} - -output "capacity_gb" { - description = "File share capacity in GiB." - value = google_filestore_instance.filestore_instance.file_shares[0].capacity_gb -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh deleted file mode 100644 index 9f842c5d7c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/install-nfs-client.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ! "$(which mount.nfs)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || - [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - major_version=$(rpm -E "%{rhel}") - enable_repo="" - if [ "${major_version}" -eq "7" ]; then - enable_repo="base,epel" - elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then - enable_repo="baseos" - else - echo "Unsupported version of centos/RHEL/Rocky" - return 1 - fi - yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils - elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get -y install nfs-common - else - echo 'Unsuported distribution' - return 1 - fi -fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/mount.sh b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/mount.sh deleted file mode 100644 index e2509fb4a1..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/mount.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -SERVER_IP=$1 -REMOTE_MOUNT=$2 -LOCAL_MOUNT=$3 -FS_TYPE=$4 -MOUNT_OPTIONS=$5 - -[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" - -if [ "${FS_TYPE}" = "gcsfuse" ]; then - FS_SPEC="${REMOTE_MOUNT}" -else - FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" -fi - -SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" -EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" - -grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false -grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false -findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/mount.yaml b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/mount.yaml deleted file mode 100644 index f7fbe58d5e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/scripts/mount.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Mounts the file systems specified in the metadata network_storage key - hosts: localhost - become: true - vars: - meta_key: "network_storage" - url: "http://metadata.google.internal/computeMetadata/v1/instance/attributes" - tasks: - - name: Read metadata network_storage information - ansible.builtin.uri: - url: "{{ url }}/{{ meta_key }}" - method: GET - headers: - Metadata-Flavor: "Google" - register: storage - - name: Mount file systems - ansible.posix.mount: - src: "{{ item.server_ip }}:/{{ item.remote_mount }}" - path: "{{ item.local_mount }}" - opts: "{{ item.mount_options }}" - boot: true - fstype: "{{ item.fs_type }}" - state: "mounted" - loop: "{{ storage.json }}" diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/variables.tf deleted file mode 100644 index 2d7e9258c0..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/variables.tf +++ /dev/null @@ -1,189 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which Filestore instance will be created." - type = string -} - -variable "deployment_name" { - description = "Name of the HPC deployment, used as name of the filestore instance if no name is specified." - type = string -} - -variable "zone" { - description = "Location for Filestore instances below Enterprise tier." - type = string -} - -variable "region" { - description = "Location for Filestore instances at Enterprise tier." - type = string -} - -variable "network_id" { - description = <<-EOT - The ID of the GCE VPC network to which the instance is connected given in the format: - `projects//global/networks/`" - EOT - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "name" { - description = "The resource name of the instance." - type = string - default = null -} - -variable "filestore_share_name" { - description = "Name of the file system share on the instance." - type = string - default = "nfsshare" -} - -variable "local_mount" { - description = "Mountpoint for this filestore instance. Note: If set to the same as the `filestore_share_name`, it will trigger a known Slurm bug ([troubleshooting](../../../docs/slurm-troubleshooting.md))." - type = string - default = "/shared" -} - -variable "size_gb" { - description = "Storage size of the filestore instance in GB." - type = number - default = 1024 - validation { - condition = var.size_gb >= 1024 - error_message = "No Filestore tier supports less than 1024GiB.\nSee https://cloud.google.com/filestore/docs/service-tiers." - } -} - -variable "filestore_tier" { - description = "The service tier of the instance." - type = string - default = "BASIC_HDD" - validation { - condition = var.filestore_tier != "STANDARD" - error_message = "The preferred name for STANDARD tier is now BASIC_HDD\nhttps://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier." - } - validation { - condition = var.filestore_tier != "PREMIUM" - error_message = "The preferred name for PREMIUM tier is now BASIC_SSD\nhttps://cloud.google.com/filestore/docs/reference/rest/v1beta1/Tier." - } - validation { - condition = contains([ - "BASIC_HDD", - "BASIC_SSD", - "HIGH_SCALE_SSD", - "ZONAL", - "REGIONAL", - "ENTERPRISE" - ], var.filestore_tier) - # Avoid adding the legacy tier name in error_message, for e.g. 'HIGH_SCALE_SSD', 'ENTERPRISE'. - # As we want to steer the customer to new one's, but also support the legacy ones for older customers. - error_message = "Allowed values for filestore_tier are 'BASIC_HDD','BASIC_SSD','ZONAL','REGIONAL'.\nhttps://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/filestore_instance#tier\nhttps://cloud.google.com/filestore/docs/reference/rest/v1/Tier." - } -} - -variable "labels" { - description = "Labels to add to the filestore instance. Key-value pairs." - type = map(string) -} - -variable "connect_mode" { - description = "Used to select mode - supported values DIRECT_PEERING and PRIVATE_SERVICE_ACCESS." - type = string - default = "DIRECT_PEERING" - nullable = false - validation { - condition = contains(["DIRECT_PEERING", "PRIVATE_SERVICE_ACCESS"], var.connect_mode) - error_message = "Allowed values for connect_mode are \"DIRECT_PEERING\" or \"PRIVATE_SERVICE_ACCESS\"." - } -} - -variable "nfs_export_options" { - description = "Define NFS export options." - type = list(object({ - access_mode = optional(string) - ip_ranges = optional(list(string)) - squash_mode = optional(string) - })) - default = [] - nullable = false -} - -variable "reserved_ip_range" { - description = <<-EOT - Reserved IP range for Filestore instance. Users are encouraged to set to null - for automatic selection. If supplied, it must be: - - CIDR format when var.connect_mode == "DIRECT_PEERING" - Named IP Range when var.connect_mode == "PRIVATE_SERVICE_ACCESS" - - See Cloud documentation for more details: - - https://cloud.google.com/filestore/docs/creating-instances#configure_a_reserved_ip_address_range - EOT - type = string - default = null - nullable = true -} - -variable "mount_options" { - description = "NFS mount options to mount file system." - type = string - default = "defaults,_netdev" -} - -variable "deletion_protection" { - description = "Configure Filestore instance deletion protection" - type = object({ - enabled = optional(bool, false) - reason = optional(string) - }) - default = { - enabled = false - } - nullable = false - - validation { - condition = !can(coalesce(var.deletion_protection.reason)) || var.deletion_protection.enabled - error_message = "Cannot set Filestore var.deletion_protection.reason unless var.deletion_protection.enabled is true" - } -} - -variable "protocol" { - description = "NFS protocol version. Default is NFS_V3. NFS_V4_1 is only supported with HIGH_SCALE_SSD, ZONAL, REGIONAL, and ENTERPRISE tiers." - type = string - default = "NFS_V3" - validation { - condition = contains(["NFS_V3", "NFS_V4_1"], var.protocol) - error_message = "Allowed values for protocol are 'NFS_V3' or 'NFS_V4_1'." - } -} - -variable "description" { - description = "A description of the filestore instance." - type = string - default = "" - validation { - condition = length(var.description) <= 2048 - error_message = "Filestore description must be 2048 characters or fewer" - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/versions.tf deleted file mode 100644 index efde3ac158..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/filestore/versions.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.4" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.57.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.57.0" - } - - required_version = ">= 1.3.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/README.md b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/README.md deleted file mode 100644 index f4d94d8c3b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/README.md +++ /dev/null @@ -1,166 +0,0 @@ -## Description - -This module creates Kubernetes Persistent Volumes (PV) and Persistent Volume -Claims (PVC) that can be used by a [gke-job-template]. - -`gke-persistent-volume` works with Filestore and Google Cloud Storage. Each -`gke-persistent-volume` can only be used with a single file system so if multiple -shared file systems are used then multiple `gke-persistent-volume` modules are -needed in the blueprint. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -### Example - -The following example creates a Filestore and then uses the -`gke-persistent-volume` module to use the Filestore as shared storage in a -`gke-job-template`. - -```yaml - - id: gke_cluster - source: modules/scheduler/gke-cluster - use: [network1] - settings: - master_authorized_networks: - - display_name: deployment-machine - cidr_block: /32 - - - id: datafs - source: modules/file-system/filestore - use: [network1] - settings: - local_mount: /data - - - id: datafs-pv - source: modules/file-system/gke-persistent-volume - use: [datafs, gke_cluster] - - - id: job-template - source: modules/compute/gke-job-template - use: [datafs-pv, compute_pool] -``` - -The following example creates a GCS bucket and then uses the -`gke-persistent-volume` module to use the bucket as shared storage in a -`gke-job-template`. - -```yaml - - id: gke_cluster - source: modules/scheduler/gke-cluster - use: [network1] - settings: - master_authorized_networks: - - display_name: deployment-machine - cidr_block: /32 - - - id: data-bucket - source: community/modules/file-system/cloud-storage-bucket - settings: - local_mount: /data - - - id: datafs-pv - source: modules/file-system/gke-persistent-volume - use: [data-bucket, gke_cluster] - - - id: job-template - source: modules/compute/gke-job-template - use: [datafs-pv, compute_pool, gke_cluster] -``` - -See example -[storage-gke.yaml](../../../../examples/README.md#storage-gkeyaml--) blueprint -for a complete example. - -### Authorized Network - -Since the `gke-persistent-volume` module is making calls to the Kubernetes API -to create Kubernetes entities, the machine performing the deployment must be -authorized to connect to the Kubernetes API. You can add the -`master_authorized_networks` settings block, as shown in the example above, with -the IP address of the machine performing the deployment. This will ensure that -the deploying machine can connect to the cluster. - -### Connecting Via Use - -The diagram below shows the valid `use` relationships for the GKE Cluster Toolkit -modules. For example the `gke-persistent-volume` module can `use` a -`gke-cluster` module and a `filestore` module, as shown in the example above. - -```mermaid -graph TD; - vpc-->|OneToMany|gke-cluster; - gke-cluster-->|OneToMany|gke-node-pool; - gke-node-pool-->|ManyToMany|gke-job-template; - gke-cluster-->|OneToMany|gke-persistent-volume; - gke-persistent-volume-->|ManyToMany|gke-job-template; - vpc-->|OneToMany|filestore; - filestore-->|OneToOne|gke-persistent-volume; -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [google](#requirement\_google) | >= 4.42 | -| [kubectl](#requirement\_kubectl) | >= 1.7.0 | -| [local](#requirement\_local) | >= 2.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.42 | -| [kubectl](#provider\_kubectl) | >= 1.7.0 | -| [local](#provider\_local) | >= 2.0.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [kubectl_manifest.pv](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.pvc](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [local_file.debug_file](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | -| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [capacity\_gb](#input\_capacity\_gb) | The storage capacity with which to create the persistent volume. | `number` | n/a | yes | -| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}` | `string` | n/a | yes | -| [filestore\_id](#input\_filestore\_id) | An identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`. | `string` | `null` | no | -| [gcs\_bucket\_name](#input\_gcs\_bucket\_name) | The gcs bucket to be used with the persistent volume. | `string` | `null` | no | -| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | -| [network\_storage](#input\_network\_storage) | Network attached storage mount to be configured. | 
object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  })
| n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [persistent\_volume\_claims](#output\_persistent\_volume\_claims) | An object that describes a k8s PVC created by this module. | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/main.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/main.tf deleted file mode 100644 index 3691523148..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/main.tf +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "gke-persistent-volume", ghpc_role = "file-system" }) -} - -locals { - is_gcs = (var.gcs_bucket_name != null) - - filestore_id = ( - !local.is_gcs ? # If not using gcs - var.filestore_id : # Then filestore_id must be provided - "projects/empty/locations/empty/instances/empty" # Otherwise use something arbitrary as it will not be used - ) - location = split("/", local.filestore_id)[3] - filestore_name = split("/", local.filestore_id)[5] - filestore_share_name = trimprefix(var.network_storage.remote_mount, "/") - base_name = local.is_gcs ? var.gcs_bucket_name : local.filestore_name - - pv_name = "${local.base_name}-pv" - pvc_name = "${local.base_name}-pvc" - - list_mount_options = split(",", var.network_storage.mount_options) - - filestore_pv_contents = templatefile( - "${path.module}/templates/filestore-pv.yaml.tftpl", - { - pv_name = local.pv_name - capacity = "${var.capacity_gb}Gi" - location = local.location - filestore_name = local.filestore_name - share_name = local.filestore_share_name - ip_address = var.network_storage.server_ip - labels = local.labels - } - ) - - filestore_pvc_contents = templatefile( - "${path.module}/templates/filestore-pvc.yaml.tftpl", - { - pv_name = local.pv_name - capacity = "${var.capacity_gb}Gi" - pvc_name = local.pvc_name - labels = local.labels - } - ) - - gcs_pv_contents = templatefile( - "${path.module}/templates/gcs-pv.yaml.tftpl", - { - pv_name = local.pv_name - capacity = "${var.capacity_gb}Gi" - labels = local.labels - mount_options = local.is_gcs ? local.list_mount_options : null - bucket_name = local.is_gcs ? var.gcs_bucket_name : "" - } - ) - - gcs_pvc_contents = templatefile( - "${path.module}/templates/gcs-pvc.yaml.tftpl", - { - pv_name = local.pv_name - pvc_name = local.pvc_name - labels = local.labels - capacity = "${var.capacity_gb}Gi" - } - ) - - cluster_name = split("/", var.cluster_id)[5] - cluster_location = split("/", var.cluster_id)[3] -} - -resource "local_file" "debug_file" { - content = <<-EOF - ${local.filestore_pv_contents} - ${local.filestore_pvc_contents} - EOF - filename = "${path.root}/pv-pvc-debug-file-${local.filestore_name}.yaml" -} - -data "google_container_cluster" "gke_cluster" { - name = local.cluster_name - location = local.cluster_location -} - -data "google_client_config" "default" {} - -provider "kubectl" { - host = "https://${data.google_container_cluster.gke_cluster.endpoint}" - cluster_ca_certificate = base64decode(data.google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate) - token = data.google_client_config.default.access_token - load_config_file = false -} - -resource "kubectl_manifest" "pv" { - yaml_body = local.is_gcs ? local.gcs_pv_contents : local.filestore_pv_contents - - lifecycle { - precondition { - condition = (var.gcs_bucket_name != null) != (var.filestore_id != null) - error_message = "Either gcs_bucket_name or filestore_id must be set." - } - } -} - -resource "kubectl_manifest" "pvc" { - yaml_body = local.is_gcs ? local.gcs_pvc_contents : local.filestore_pvc_contents - depends_on = [kubectl_manifest.pv] -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf deleted file mode 100644 index 3c7d1b7050..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/outputs.tf +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "persistent_volume_claims" { - description = "An object that describes a k8s PVC created by this module." - value = { - name = local.pvc_name - mount_path = var.network_storage.local_mount - mount_options = var.network_storage.mount_options - is_gcs = local.is_gcs - } - depends_on = [kubectl_manifest.pvc] -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl deleted file mode 100644 index cfda33978c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pv.yaml.tftpl +++ /dev/null @@ -1,23 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: ${pv_name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - storageClassName: "" - capacity: - storage: ${capacity} - accessModes: - - ReadWriteMany - persistentVolumeReclaimPolicy: Retain - volumeMode: Filesystem - csi: - driver: filestore.csi.storage.gke.io - volumeHandle: "modeInstance/${location}/${filestore_name}/${share_name}" - volumeAttributes: - ip: ${ip_address} - volume: ${share_name} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl deleted file mode 100644 index 5bcd735807..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/filestore-pvc.yaml.tftpl +++ /dev/null @@ -1,17 +0,0 @@ ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: ${pvc_name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ReadWriteMany - storageClassName: "" - volumeName: ${pv_name} - resources: - requests: - storage: ${capacity} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl deleted file mode 100644 index aa0e570a8b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pv.yaml.tftpl +++ /dev/null @@ -1,24 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: ${pv_name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - storageClassName: "" - capacity: - storage: ${capacity} - accessModes: - - ReadWriteMany - %{~ if mount_options != null ~} - mountOptions: - %{~ for key in mount_options ~} - - ${key} - %{~ endfor ~} - %{~ endif ~} - csi: - driver: gcsfuse.csi.storage.gke.io - volumeHandle: ${bucket_name} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl deleted file mode 100644 index 9af95b29d7..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/templates/gcs-pvc.yaml.tftpl +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ${pvc_name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ReadWriteMany - storageClassName: "" - volumeName: ${pv_name} - resources: - requests: - storage: ${capacity} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf deleted file mode 100644 index a72fa3857f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/variables.tf +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "cluster_id" { - description = "An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}`" - type = string -} - -variable "network_storage" { - description = "Network attached storage mount to be configured." - type = object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string, - client_install_runner = map(string) - mount_runner = map(string) - }) -} - -variable "filestore_id" { - description = "An identifier for a filestore with the format `projects/{{project}}/locations/{{location}}/instances/{{name}}`." - type = string - default = null - validation { - condition = ( - var.filestore_id == null || - try(length(split("/", var.filestore_id)), 0) == 6 - ) - error_message = "filestore_id must be in the format of 'projects/{{project}}/locations/{{location}}/instances/{{name}}'." - } -} - -variable "gcs_bucket_name" { - description = "The gcs bucket to be used with the persistent volume." - type = string - default = null -} - -variable "capacity_gb" { - description = "The storage capacity with which to create the persistent volume." - type = number -} - -variable "labels" { - description = "GCE resource labels to be applied to resources. Key-value pairs." - type = map(string) -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf deleted file mode 100644 index 1c6b08e279..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-persistent-volume/versions.tf +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.0" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.42" - } - kubectl = { - source = "gavinbunney/kubectl" - version = ">= 1.7.0" - } - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-persistent-volume/v1.57.0" - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/README.md b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/README.md deleted file mode 100644 index 73396f6a03..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/README.md +++ /dev/null @@ -1,133 +0,0 @@ -## Description - -This module creates Kubernetes Storage Class (SC) that can be used by a Persistent Volume Claim (PVC) -to dynamically provision GCP storage resources like Parallelstore. - -### Example - -The following example uses the `gke-storage` module to creates a Parallelstore Storage Class and Persistent Volume Claim, -then use them in a `gke-job-template` to dynamically provision the resource. - -```yaml - - id: gke_cluster - source: modules/scheduler/gke-cluster - use: [network] - settings: - enable_parallelstore_csi: true - - # Private Service Access (PSA) requires the compute.networkAdmin role which is - # included in the Owner role, but not Editor. - # PSA is required for all Parallelstore functionality. - # https://cloud.google.com/vpc/docs/configure-private-services-access#permissions - - id: private_service_access - source: community/modules/network/private-service-access - use: [network] - settings: - prefix_length: 24 - - - id: gke_storage - source: modules/file-system/gke-storage - use: [ gke_cluster, private_service_access ] - settings: - storage_type: Parallelstore - access_mode: ReadWriteMany - sc_volume_binding_mode: Immediate - sc_reclaim_policy: Delete - sc_topology_zones: [$(vars.zone)] - pvc_count: 2 - capacity_gb: 12000 - - - id: job_template - source: modules/compute/gke-job-template - use: [gke_storage, compute_pool] -``` - -See example -[gke-managed-parallelstore.yaml](../../../examples/README.md#gke-managed-parallelstoreyaml--) blueprint -for a complete example. - -### Authorized Network - -Since the `gke-storage` module is making calls to the Kubernetes API -to create Kubernetes entities, the machine performing the deployment must be -authorized to connect to the Kubernetes API. You can add the -`master_authorized_networks` settings block, as shown in the example above, with -the IP address of the machine performing the deployment. This will ensure that -the deploying machine can connect to the cluster. - -### Connecting Via Use - -The diagram below shows the valid `use` relationships for the GKE Cluster Toolkit -modules. For example the `gke-storage` module can `use` a -`gke-cluster` module and a `private_service_access` module, as shown in the example above. - -```mermaid -graph TD; - vpc-->|OneToMany|gke-cluster; - gke-cluster-->|OneToMany|gke-node-pool; - gke-node-pool-->|ManyToMany|gke-job-template; - gke-cluster-->|OneToMany|gke-storage; - gke-storage-->|ManyToMany|gke-job-template; -``` - -## License - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [access\_mode](#input\_access\_mode) | The access mode that the volume can be mounted to the host/pod. More details in [Access Modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes)
Valid access modes:
- ReadWriteOnce
- ReadOnlyMany
- ReadWriteMany
- ReadWriteOncePod | `string` | n/a | yes | -| [capacity\_gb](#input\_capacity\_gb) | The storage capacity with which to create the persistent volume. | `number` | n/a | yes | -| [cluster\_id](#input\_cluster\_id) | An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}` | `string` | n/a | yes | -| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | -| [mount\_options](#input\_mount\_options) | Controls the mountOptions for dynamically provisioned PersistentVolumes of this storage class. | `string` | `null` | no | -| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). | `string` | `null` | no | -| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | -| [pv\_mount\_path](#input\_pv\_mount\_path) | Path within the container at which the volume should be mounted. Must not contain ':'. | `string` | `"/data"` | no | -| [pvc\_count](#input\_pvc\_count) | How many PersistentVolumeClaims that will be created | `number` | `1` | no | -| [sc\_reclaim\_policy](#input\_sc\_reclaim\_policy) | Indicate whether to keep the dynamically provisioned PersistentVolumes of this storage class after the bound PersistentVolumeClaim is deleted.
[More details about reclaiming](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming)
Supported value:
- Retain
- Delete | `string` | n/a | yes | -| [sc\_topology\_zones](#input\_sc\_topology\_zones) | Zone location that allow the volumes to be dynamically provisioned. | `list(string)` | `null` | no | -| [sc\_volume\_binding\_mode](#input\_sc\_volume\_binding\_mode) | Indicates when volume binding and dynamic provisioning should occur and how PersistentVolumeClaims should be provisioned and bound.
Supported value:
- Immediate
- WaitForFirstConsumer | `string` | `"WaitForFirstConsumer"` | no | -| [storage\_type](#input\_storage\_type) | The type of [GKE supported storage options](https://cloud.google.com/kubernetes-engine/docs/concepts/storage-overview)
to used. This module currently support dynamic provisioning for the below storage options
- Parallelstore
- Hyperdisk-balanced
- Hyperdisk-throughput
- Hyperdisk-extreme | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [persistent\_volume\_claims](#output\_persistent\_volume\_claims) | An object that describes a k8s PVC created by this module. | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/main.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/main.tf deleted file mode 100644 index 18f85fa779..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/main.tf +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "gke-storage", ghpc_role = "file-system" }) -} - -locals { - storage_type = lower(var.storage_type) - storage_class_name = "${local.storage_type}-sc" - pvc_name_prefix = "${local.storage_type}-pvc" -} - -check "private_vpc_connection_peering" { - assert { - condition = lower(var.storage_type) != "parallelstore" ? true : var.private_vpc_connection_peering != null - error_message = <<-EOT - Parallelstore must be run within the same VPC as the GKE cluster and have private services access enabled. - If using new VPC, please use community/modules/network/private-service-access to create private-service-access. - If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). - EOT - } -} - -module "kubectl_apply" { - source = "../../management/kubectl-apply" - - cluster_id = var.cluster_id - project_id = var.project_id - - # count = var.pvc_count - apply_manifests = flatten( - [ - # create StorageClass in the cluster - { - content = templatefile( - "${path.module}/storage-class/${local.storage_class_name}.yaml.tftpl", - { - name = local.storage_class_name - labels = local.labels - volume_binding_mode = var.sc_volume_binding_mode - reclaim_policy = var.sc_reclaim_policy - topology_zones = var.sc_topology_zones - }) - }, - # create PersistentVolumeClaim in the cluster - flatten([ - for idx in range(var.pvc_count) : [ - { - content = templatefile( - "${path.module}/persistent-volume-claim/${(local.pvc_name_prefix)}.yaml.tftpl", - { - pvc_name = "${local.pvc_name_prefix}-${idx}" - labels = local.labels - capacity = "${var.capacity_gb}Gi" - access_mode = var.access_mode - storage_class_name = local.storage_class_name - } - ) - } - ] - ]) - ]) -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/metadata.yaml deleted file mode 100644 index 8722823274..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/outputs.tf deleted file mode 100644 index b789674814..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/outputs.tf +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "persistent_volume_claims" { - description = "An object that describes a k8s PVC created by this module." - value = flatten([ - for idx in range(var.pvc_count) : [{ - name = "${local.pvc_name_prefix}-${idx}" - mount_path = "${var.pv_mount_path}/${local.pvc_name_prefix}-${idx}" - mount_options = var.mount_options - is_gcs = false - }] - ]) -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl deleted file mode 100644 index 32781be2fb..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-balanced-pvc.yaml.tftpl +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ${pvc_name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ${access_mode} - resources: - requests: - storage: ${capacity} - storageClassName: ${storage_class_name} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl deleted file mode 100644 index 32781be2fb..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-extreme-pvc.yaml.tftpl +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ${pvc_name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ${access_mode} - resources: - requests: - storage: ${capacity} - storageClassName: ${storage_class_name} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl deleted file mode 100644 index 32781be2fb..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/hyperdisk-throughput-pvc.yaml.tftpl +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ${pvc_name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ${access_mode} - resources: - requests: - storage: ${capacity} - storageClassName: ${storage_class_name} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl deleted file mode 100644 index 32781be2fb..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/persistent-volume-claim/parallelstore-pvc.yaml.tftpl +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ${pvc_name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -spec: - accessModes: - - ${access_mode} - resources: - requests: - storage: ${capacity} - storageClassName: ${storage_class_name} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl deleted file mode 100644 index 46e1f023d3..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-balanced-sc.yaml.tftpl +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: ${name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -provisioner: pd.csi.storage.gke.io -allowVolumeExpansion: true -parameters: - type: hyperdisk-balanced - provisioned-throughput-on-create: "250Mi" - provisioned-iops-on-create: "7000" -volumeBindingMode: ${volume_binding_mode} -reclaimPolicy: ${reclaim_policy} - %{~ if topology_zones != null ~} -allowedTopologies: -- matchLabelExpressions: - - key: topology.gke.io/zone - values: - %{~ for z in topology_zones ~} - - ${z} - %{~ endfor ~} - %{~ endif ~} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl deleted file mode 100644 index 445020d001..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-extreme-sc.yaml.tftpl +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: ${name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} -provisioner: pd.csi.storage.gke.io -allowVolumeExpansion: true -parameters: - %{~ endfor ~} - type: hyperdisk-extreme - provisioned-iops-on-create: "50000" -volumeBindingMode: ${volume_binding_mode} -reclaimPolicy: ${reclaim_policy} - %{~ if topology_zones != null ~} -allowedTopologies: -- matchLabelExpressions: - - key: topology.gke.io/zone - values: - %{~ for z in topology_zones ~} - - ${z} - %{~ endfor ~} - %{~ endif ~} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl deleted file mode 100644 index ec404aec45..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/hyperdisk-throughput-sc.yaml.tftpl +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: ${name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -provisioner: pd.csi.storage.gke.io -allowVolumeExpansion: true -parameters: - type: hyperdisk-throughput - provisioned-throughput-on-create: "250Mi" -volumeBindingMode: ${volume_binding_mode} -reclaimPolicy: ${reclaim_policy} - %{~ if topology_zones != null ~} -allowedTopologies: -- matchLabelExpressions: - - key: topology.gke.io/zone - values: - %{~ for z in topology_zones ~} - - ${z} - %{~ endfor ~} - %{~ endif ~} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl deleted file mode 100644 index e6b8ea8d3e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/storage-class/parallelstore-sc.yaml.tftpl +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: ${name} - labels: - %{~ for key, val in labels ~} - ${key}: ${val} - %{~ endfor ~} -provisioner: parallelstore.csi.storage.gke.io -parameters: -volumeBindingMode: ${volume_binding_mode} -reclaimPolicy: ${reclaim_policy} - %{~ if topology_zones != null ~} -allowedTopologies: -- matchLabelExpressions: - - key: topology.gke.io/zone - values: - %{~ for z in topology_zones ~} - - ${z} - %{~ endfor ~} - %{~ endif ~} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/variables.tf deleted file mode 100644 index 9f6224bbac..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/variables.tf +++ /dev/null @@ -1,138 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "The project ID to host the cluster in." - type = string -} - -variable "cluster_id" { - description = "An identifier for the GKE cluster in the format `projects/{{project}}/locations/{{location}}/clusters/{{cluster}}`" - type = string -} - -variable "labels" { - description = "GCE resource labels to be applied to resources. Key-value pairs." - type = map(string) -} - -variable "storage_type" { - description = <<-EOT - The type of [GKE supported storage options](https://cloud.google.com/kubernetes-engine/docs/concepts/storage-overview) - to used. This module currently support dynamic provisioning for the below storage options - - Parallelstore - - Hyperdisk-balanced - - Hyperdisk-throughput - - Hyperdisk-extreme - EOT - type = string - nullable = false - validation { - condition = var.storage_type == null ? false : contains(["parallelstore", "hyperdisk-balanced", "hyperdisk-throughput", "hyperdisk-extreme"], lower(var.storage_type)) - error_message = "Allowed string values for var.storage_type are \"Parallelstore\", \"Hyperdisk-balanced\", \"Hyperdisk-throughput\", \"Hyperdisk-extreme\"." - } -} - -variable "access_mode" { - description = <<-EOT - The access mode that the volume can be mounted to the host/pod. More details in [Access Modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) - Valid access modes: - - ReadWriteOnce - - ReadOnlyMany - - ReadWriteMany - - ReadWriteOncePod - EOT - type = string - nullable = false - validation { - condition = var.access_mode == null ? false : contains(["readwriteonce", "readonlymany", "readwritemany", "readwriteoncepod"], lower(var.access_mode)) - error_message = "Allowed string values for var.access_mode are \"ReadWriteOnce\", \"ReadOnlyMany\", \"ReadWriteMany\", \"ReadWriteOncePod\"." - } -} - -variable "sc_volume_binding_mode" { - description = <<-EOT - Indicates when volume binding and dynamic provisioning should occur and how PersistentVolumeClaims should be provisioned and bound. - Supported value: - - Immediate - - WaitForFirstConsumer - EOT - type = string - default = "WaitForFirstConsumer" - validation { - condition = var.sc_volume_binding_mode == null ? true : contains(["immediate", "waitforfirstconsumer"], lower(var.sc_volume_binding_mode)) - error_message = "Allowed string values for var.sc_volume_binding_mode are \"Immediate\", \"WaitForFirstConsumer\"." - } -} - -variable "sc_reclaim_policy" { - description = <<-EOT - Indicate whether to keep the dynamically provisioned PersistentVolumes of this storage class after the bound PersistentVolumeClaim is deleted. - [More details about reclaiming](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming) - Supported value: - - Retain - - Delete - EOT - type = string - nullable = false - validation { - condition = var.sc_reclaim_policy == null ? true : contains(["retain", "delete"], lower(var.sc_reclaim_policy)) - error_message = "Allowed string values for var.sc_reclaim_policy are \"Retain\", \"Delete\"." - } -} - -variable "sc_topology_zones" { - description = "Zone location that allow the volumes to be dynamically provisioned." - type = list(string) - default = null -} - -variable "pvc_count" { - description = "How many PersistentVolumeClaims that will be created" - type = number - default = 1 -} - -variable "pv_mount_path" { - description = "Path within the container at which the volume should be mounted. Must not contain ':'." - type = string - default = "/data" - validation { - condition = var.pv_mount_path == null ? true : !strcontains(var.pv_mount_path, ":") - error_message = "pv_mount_path must not contain ':', please correct it and retry" - } -} - -variable "mount_options" { - description = "Controls the mountOptions for dynamically provisioned PersistentVolumes of this storage class." - type = string - default = null -} - -variable "capacity_gb" { - description = "The storage capacity with which to create the persistent volume." - type = number -} - -variable "private_vpc_connection_peering" { - description = <<-EOT - The name of the VPC Network peering connection. - If using new VPC, please use community/modules/network/private-service-access to create private-service-access and - If using existing VPC with private-service-access enabled, set this manually follow [user guide](https://cloud.google.com/parallelstore/docs/vpc). - EOT - type = string - default = null -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/versions.tf deleted file mode 100644 index ab0ea408ce..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/gke-storage/versions.tf +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.5" - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:gke-storage/v1.57.0" - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/README.md b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/README.md deleted file mode 100644 index e9c7f66268..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/README.md +++ /dev/null @@ -1,289 +0,0 @@ -## Description - -This module creates a [Managed Lustre](https://cloud.google.com/managed-lustre) -instance. Managed Lustre is a high performance network file system that can be -mounted to one or more VMs. - -For more information on this and other network storage options in the Cluster -Toolkit, see the extended [Network Storage documentation](../../../docs/network_storage.md). - -### Supported Operating Systems - -A Managed Lustre instance can be used with Slurm cluster or compute -VM running Ubuntu 20.04, 22.04 or Rocky Linux 8 (including the HPC flavor). - -### Managed Lustre Access - -Managed Lustre must be enabled for your project by Google staff. Please contact -your sales representative for further steps. - -### Example - New VPC - -For Managed Lustre instance, the snippet below creates new VPC and configures -private-service-access for this newly created network. Both items are required -to be passed to the Lustre module to ensure that they're built in order and -that the correct subnetwork has private service access. - -```yaml - - id: network - source: modules/network/vpc - - - id: private_service_access - source: community/modules/network/private-service-access - use: [network] - settings: - prefix_length: 24 - - - id: lustre - source: modules/file-system/managed-lustre - use: [network, private_service_access] -``` - -### Example - Slurm - -When using Slurm you must take into consideration whether or not you are using -an official image from the `schedmd-slurm-public` project or building your own. -The Lustre client modules are pre-installed in the official images. With the -official images, Lustre can be used as follows: - -```yaml -- id: managed_lustre - source: modules/file-system/managed-lustre - use: [network, private_service_access] - settings: - name: lustre-instance - local_mount: /lustre - remote_mount: lustrefs - size_gib: 18000 - -# Other modules: nodesets, partitions, login, etc. - -- id: slurm_controller - source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller - use: - - network - - lustre_partition - - managed_lustre - - slurm_login - settings: - machine_type: n2-standard-4 - enable_controller_public_ips: true -``` - -For custom images you must install the modules during the image build as the -Slurm cluster will not run the installation script like it does for the -standard VMs. - -Assuming you have a startup script for the Slurm image building, you can add -this Ansible playbook to correctly install the Lustre drivers into the image -(for Slurm-GCP versions greater than 6.10.0): - -```yaml -- type: data - destination: /var/tmp/slurm_vars.json - content: | - { - "reboot": false, - "install_cuda": false, - "install_gcsfuse": true, - "install_lustre": false, - "install_managed_lustre": true, - "install_nvidia_repo": true, - "install_ompi": true, - "allow_kernel_upgrades": false, - "monitoring_agent": "cloud-ops", - } -``` - -The `install_managed_lustre: true` line specifies that slurm-gcp should install -the correct modules within the slurm image. This runner should be placed -ahead of the script that calls the ansible build of the slurm-gcp image. - -### Example - Existing VPC - -If you want to use existing network with private-service-access configured, you need -to manually provide `private_vpc_connection_peering` to the Managed Lustre module. -You can get this details from the Google Cloud Console UI in `VPC network peering` -section. Below is the example of using existing network and creating Managed Lustre. -If existing network is not configured with private-service-access, you can follow -[Configure private service access](https://cloud.google.com/vpc/docs/configure-private-services-access) -to set it up. - -```yaml - - id: network - source: modules/network/pre-existing-vpc - settings: - network_name: // Add network name - subnetwork_name: // Add subnetwork name - - - id: lustre - source: modules/file-system/managed-lustre - use: [network] - settings: - private_vpc_connection_peering: # will look like "servicenetworking.googleapis.com" -``` - -### Example - GKE compatibility - -By default the Managed Lustre instance that is deployed is not compatible with -GKE. To enable the compatibility use the `gke_support_enabled: true` option. -This creates a file `/etc/modprobe/lnet.conf` that changes the listening port -to 6988. - -```yaml - - id: managed-lustre - source: modules/file-system/managed-lustre - use: [network, private_service_access] - settings: - name: lustre-instance - local_mount: /lustre - remote_mount: lustrefs - size_gib: 18000 - gke_support_enabled: true -``` - -> [!WARNING] -> -> 1. VMs cannot connect to both GKE compatible and GKE incompatible lustre -> instances at the same time as they connect to different ports. Lustre can -> only listen to one port at a time. -> -> 2. Setting `gke_support_enabled: true` will not affect Slurm nodes, GKE -> compatibility must be built into the Slurm image. - -### Example - Importing data from GSC Bucket - -One option with the Managed Lustre instance is to import data from a GSC bucket -upon the lustre instance creation. To do this, use the `import_gcs_bucket_uri` -variable to dictate the bucket to pull data from. The data will be imported -under the directory specified by `local_mount` (`/shared` if unspecified). - -> [!NOTE] -> -> 1. This is a one way operation. Once the data has been copied to the lustre -> instance it will not be updated with any changes made to the GCS bucket. -> -> 2. Once the lustre instance has been created in Terraform, the copy process -> will proceed in the background. Data may not be appear in the mounted -> directory for a period of time after the deployment has completed (see below). - -```yaml -- id: managed_lustre - source: modules/file-system/managed-lustre - use: [network, private_service_access] - settings: - name: lustre-instance - local_mount: /lustre - remote_mount: lustrefs - size_gib: 18000 - import_gcs_bucket_uri: gs:// -``` - -> [!WARNING] -> Please follow [this guide](https://cloud.google.com/managed-lustre/docs/transfer-data#required_permissions) -> to set up the correct IAM permissions for importing data from GCS to lustre. -> Without this, the copy process may fail silently leaving an empty lustre -> instance. - -If an import is requested, gcluster will output a json response similar to: - -```json -{ - "name": "projects//locations//operations/", - "metadata": { - "@type": "type.googleapis.com/google.cloud.lustre.v1.ImportDataMetadata", - "createTime": "", - "target": "projects//locations//instances/", - "requestedCancellation": false, - "apiVersion": "v1" - }, - "done": false -} -``` - -You can retrieve more information about the transfer using the following -command, substituting with values from the json response above: - -```bash -gcloud lustre operations describe --location --project -``` - -This will provide information on if the transfer is complete or if any errors -have occurred. See more at -[Get operation](https://cloud.google.com/managed-lustre/docs/transfer-data#get_operation). - -## License - - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [google](#requirement\_google) | >= 6.27.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.27.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_lustre_instance.lustre_instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/lustre_instance) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [google_compute_network_peering.private_peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network_peering) | data source | -| [google_storage_bucket.lustre_import_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/storage_bucket) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the Lustre instance if no name is specified. | `string` | n/a | yes | -| [description](#input\_description) | Description of the created Lustre instance. | `string` | `"Lustre Instance"` | no | -| [gke\_support\_enabled](#input\_gke\_support\_enabled) | Set to true to create Managed Lustre instance with GKE compatibility.
Note: This does not work with Slurm, the Slurm image must be built with
the correct compatibility. | `bool` | `false` | no | -| [import\_gcs\_bucket\_uri](#input\_import\_gcs\_bucket\_uri) | The name of the GCS bucket to import data from to managed lustre. Data will
be imported to the local\_mount directory. Changing this value will not
trigger a redeployment, to prevent data deletion. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to the Managed Lustre instance. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_mount](#input\_local\_mount) | Local mount point for the Managed Lustre instance. | `string` | `"/shared"` | no | -| [mount\_options](#input\_mount\_options) | Mounting options for the file system. | `string` | `"defaults,_netdev"` | no | -| [name](#input\_name) | Name of the Lustre instance | `string` | n/a | yes | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | -| [network\_self\_link](#input\_network\_self\_link) | Network self-link this instance will be on, required for checking private service access | `string` | n/a | yes | -| [per\_unit\_storage\_throughput](#input\_per\_unit\_storage\_throughput) | Throughput of the instance in MB/s/TiB. Valid values are 125, 250, 500, 1000. | `number` | `1000` | no | -| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually." | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | ID of project in which Lustre instance will be created. | `string` | n/a | yes | -| [remote\_mount](#input\_remote\_mount) | Remote mount point of the Managed Lustre instance | `string` | n/a | yes | -| [size\_gib](#input\_size\_gib) | Storage size of the Managed Lustre instance in GB. See https://cloud.google.com/managed-lustre/docs/create-instance for limitations | `number` | `18000` | no | -| [zone](#input\_zone) | Location for the Lustre instance. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [capacity\_gib](#output\_capacity\_gib) | File share capacity in GiB. | -| [install\_managed\_lustre\_client](#output\_install\_managed\_lustre\_client) | Script for installing Managed Lustre client | -| [lustre\_id](#output\_lustre\_id) | An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}` | -| [network\_storage](#output\_network\_storage) | Describes a Managed Lustre instance. | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/main.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/main.tf deleted file mode 100644 index a969c53673..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/main.tf +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "managed-lustre", ghpc_role = "file-system" }) -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -data "google_compute_network_peering" "private_peering" { - name = var.private_vpc_connection_peering - network = var.network_self_link -} - -locals { - server_ip = split(":", google_lustre_instance.lustre_instance.mount_point)[0] - remote_mount = split(":", google_lustre_instance.lustre_instance.mount_point)[1] - fs_type = "lustre" - mount_options = var.mount_options - instance_id = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" - destination_path = "/" - - install_managed_lustre_client_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/install-managed-lustre-client.sh" - "destination" = "install-managed-lustre-client${replace(var.local_mount, "/", "_")}.sh" - "args" = var.gke_support_enabled ? "1" : "0" - } - mount_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/mount.sh" - "args" = "\"${local.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${local.mount_options}\"" - "destination" = "mount${replace(var.local_mount, "/", "_")}.sh" - } - - bucket_count = try(length(data.google_storage_bucket.lustre_import_bucket), 0) -} - -data "google_storage_bucket" "lustre_import_bucket" { - count = try(length(var.import_gcs_bucket_uri) > 0, false) ? 1 : 0 - - name = split("//", var.import_gcs_bucket_uri)[1] -} - -resource "google_lustre_instance" "lustre_instance" { - project = var.project_id - - description = var.description - instance_id = local.instance_id - location = var.zone - - filesystem = var.remote_mount - capacity_gib = var.size_gib - per_unit_storage_throughput = var.per_unit_storage_throughput - - labels = local.labels - network = var.network_id - - gke_support_enabled = var.gke_support_enabled - - timeouts { - create = "1h" - update = "1h" - delete = "1h" - } - - depends_on = [var.private_vpc_connection_peering, data.google_storage_bucket.lustre_import_bucket] - - lifecycle { - precondition { - condition = data.google_compute_network_peering.private_peering.state == "ACTIVE" - error_message = "The subnetwork that the lustre instance is hosted on must have private service access." - } - } - - provisioner "local-exec" { - command = < 0 ]]; then - curl -X POST \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $(gcloud auth print-access-token)" \ - -d '{"gcsPath": {"uri":"${coalesce(var.import_gcs_bucket_uri, "gs://")}"}, "lustrePath": {"path":"${local.destination_path}"}}' \ - https://lustre.googleapis.com/v1/projects/${var.project_id}/locations/${var.zone}/instances/${local.instance_id}:importData - fi - EOF - interpreter = ["bash", "-c"] - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/metadata.yaml deleted file mode 100644 index 66da9827b6..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - lustre.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/outputs.tf deleted file mode 100644 index fdc0fb20e1..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/outputs.tf +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "network_storage" { - description = "Describes a Managed Lustre instance." - value = { - server_ip = local.server_ip - remote_mount = local.remote_mount - local_mount = var.local_mount - fs_type = local.fs_type - mount_options = local.mount_options - client_install_runner = local.install_managed_lustre_client_runner - mount_runner = local.mount_runner - } -} - -output "install_managed_lustre_client" { - description = "Script for installing Managed Lustre client" - value = file("${path.module}/scripts/install-managed-lustre-client.sh") -} - -output "lustre_id" { - description = "An identifier for the resource with format `projects/{{project}}/locations/{{location}}/instances/{{name}}`" - value = google_lustre_instance.lustre_instance.instance_id -} - -output "capacity_gib" { - description = "File share capacity in GiB." - value = google_lustre_instance.lustre_instance.capacity_gib -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh deleted file mode 100644 index 878130ab47..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/scripts/install-managed-lustre-client.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Install Managed Lustre client modules -# Based on these instructions: https://cloud.google.com/managed-lustre/docs/connect-from-compute-engine - -# The client modules currently only support Rocky 8, and Ubuntu 20.04/22.04 - -set -e - -GKE_ENABLED=$1 - -# Update lnet to enable GKE supported Lustre instance -if [[ $GKE_ENABLED == "1" ]]; then - if [[ -f "/etc/modprobe.d/lnet.conf" ]] && grep -Fq "options lnet accept_port" /etc/modprobe.d/lnet.conf; then - echo "Lnet accept port already set, continuing without updating /etc/modprobe.d/lnet.conf" - else - echo "options lnet accept_port=6988" >>/etc/modprobe.d/lnet.conf - fi -fi - -if grep -q lustre /proc/filesystems; then - echo "Skipping managed lustre client install as it is already supported" - exit 0 -fi - -# Get distro information -. /etc/os-release -DIST="NA" -if [[ $NAME == *"Ubuntu"* ]]; then - if [[ $VERSION_ID == "20.04" || $VERSION_ID == "22.04" ]]; then - DIST="Ubuntu" - fi -elif [[ $NAME == *"Rocky"* ]]; then - if [[ $VERSION_ID == "8"* ]]; then - DIST="Rocky" - fi -fi - -if [[ ${DIST} == "Ubuntu" ]]; then - KEY_LOC=/etc/apt/keyrings - KEY_NAME=gcp-ar-repo.gpg - # Download new repo key - mkdir -p "${KEY_LOC}" - wget -O - https://us-apt.pkg.dev/doc/repo-signing-key.gpg 2>/dev/null | gpg --dearmor - | tee "${KEY_LOC}/${KEY_NAME}" >/dev/null - - # Set up apt repo - echo "deb [ signed-by=${KEY_LOC}/${KEY_NAME} ] https://us-apt.pkg.dev/projects/lustre-client-binaries lustre-client-ubuntu-${UBUNTU_CODENAME} main" | tee -a /etc/apt/sources.list.d/artifact-registry.list - - # Install modules - apt update - apt install -y "lustre-client-modules-$(uname -r)" lustre-client-utils || (echo "Error finding Lustre module packages, Lustre package may not exist for this kernel version" && exit 1) -elif [[ ${DIST} == "Rocky" ]]; then - # Set up yum repo - touch /etc/yum.repos.d/artifact-registry.repo - tee -a /etc/yum.repos.d/artifact-registry.repo <<-EOF - [lustre-client-rocky-8] - name=lustre-client-rocky-8 - baseurl=https://us-yum.pkg.dev/projects/lustre-client-binaries/lustre-client-rocky-8 - enabled=1 - repo_gpgcheck=0 - gpgcheck=0 - EOF - # Install modules - yum makecache - yum --enablerepo=lustre-client-rocky-8 install -y kmod-lustre-client lustre-client -fi - -if [[ $DIST != "NA" ]]; then - # Load the new lustre client module - modprobe lustre -fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh deleted file mode 100644 index e2509fb4a1..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/scripts/mount.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -SERVER_IP=$1 -REMOTE_MOUNT=$2 -LOCAL_MOUNT=$3 -FS_TYPE=$4 -MOUNT_OPTIONS=$5 - -[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" - -if [ "${FS_TYPE}" = "gcsfuse" ]; then - FS_SPEC="${REMOTE_MOUNT}" -else - FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" -fi - -SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" -EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" - -grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false -grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false -findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/variables.tf deleted file mode 100644 index aa5d3be0ae..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/variables.tf +++ /dev/null @@ -1,131 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "ID of project in which Lustre instance will be created." - type = string -} - -variable "description" { - description = "Description of the created Lustre instance." - type = string - default = "Lustre Instance" -} - -variable "deployment_name" { - description = "Name of the HPC deployment, used as name of the Lustre instance if no name is specified." - type = string -} - -variable "zone" { - description = "Location for the Lustre instance." - type = string -} - -variable "name" { - description = "Name of the Lustre instance" - type = string -} - -variable "network_id" { - description = <<-EOT - The ID of the GCE VPC network to which the instance is connected given in the format: - `projects//global/networks/`" - EOT - type = string - nullable = false - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "network_self_link" { - description = "Network self-link this instance will be on, required for checking private service access" - type = string - nullable = false -} - -variable "remote_mount" { - description = "Remote mount point of the Managed Lustre instance" - type = string - nullable = false -} - -variable "local_mount" { - description = "Local mount point for the Managed Lustre instance." - type = string - default = "/shared" -} - -variable "size_gib" { - description = "Storage size of the Managed Lustre instance in GB. See https://cloud.google.com/managed-lustre/docs/create-instance for limitations" - type = number - default = 18000 -} - -variable "per_unit_storage_throughput" { - description = "Throughput of the instance in MB/s/TiB. Valid values are 125, 250, 500, 1000." - type = number - default = 1000 -} - -variable "labels" { - description = "Labels to add to the Managed Lustre instance. Key-value pairs." - type = map(string) -} - -variable "mount_options" { - description = "Mounting options for the file system." - type = string - default = "defaults,_netdev" -} - -variable "private_vpc_connection_peering" { - description = <<-EOT - The name of the VPC Network peering connection. - If using new VPC, please use community/modules/network/private-service-access to create private-service-access and - If using existing VPC with private-service-access enabled, set this manually." - EOT - type = string - nullable = false -} - -variable "gke_support_enabled" { - description = <<-EOT - Set to true to create Managed Lustre instance with GKE compatibility. - Note: This does not work with Slurm, the Slurm image must be built with - the correct compatibility. - EOT - type = bool - nullable = false - default = false -} - -variable "import_gcs_bucket_uri" { - description = <<-EOT - The name of the GCS bucket to import data from to managed lustre. Data will - be imported to the local_mount directory. Changing this value will not - trigger a redeployment, to prevent data deletion. - EOT - type = string - default = null - - validation { - condition = startswith(coalesce(var.import_gcs_bucket_uri, "gs://"), "gs://") - error_message = "The GCS bucket uri must start with 'gs://'" - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/versions.tf deleted file mode 100644 index 59808977a6..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/managed-lustre/versions.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.27.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.57.0" - } - provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:managed-lustre/v1.57.0" - } - - required_version = ">= 1.3.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/README.md b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/README.md deleted file mode 100644 index 0b942f067f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/README.md +++ /dev/null @@ -1,196 +0,0 @@ -## Description - -This module creates [parallelstore](https://cloud.google.com/parallelstore) -instance. Parallelstore is Google Cloud's first party parallel file system -service based on [Intel DAOS](https://docs.daos.io/v2.2/) - -### Supported Operating Systems - -A parallelstore instance can be used with Slurm cluster or compute -VM running Ubuntu 22.04, debian 12 or HPC Rocky Linux 8. - -### Parallelstore Quota - -To get access to a private preview of Parallelstore APIs, your project needs to -be allowlisted. To set this up, please work with your account representative. - -### Parallelstore mount options - -After parallelstore instance is created, you can specify mount options depending -upon your workload. DAOS is configured to deliver the best user experience for -interactive workloads with aggressive caching. If you are running parallel -workloads concurrently accessing the sane files from multiple client nodes, it -is recommended to disable the writeback cache to avoid cross-client consistency -issues. You can specify different mount options as follows, - -```yaml - - id: parallelstore - source: modules/file-system/parallelstore - use: [network, ps_connect] - settings: - mount_options: "disable-wb-cache,thread-count=20,eq-count=8" -``` - -### Example - New VPC - -For parallelstore instance, Below snippet creates new VPC and configures private-service-access -for this newly created network. - -```yaml - - id: network - source: modules/network/vpc - - # Private Service Access (PSA) requires the compute.networkAdmin role which is - # included in the Owner role, but not Editor. - # PSA is required for all Parallelstore functionality. - # https://cloud.google.com/vpc/docs/configure-private-services-access#permissions - - id: private_service_access - source: community/modules/network/private-service-access - use: [network] - settings: - prefix_length: 24 - - - id: parallelstore - source: modules/file-system/parallelstore - use: [network, private_service_access] -``` - -### Example - Existing VPC - -If you want to use existing network with private-service-access configured, you need -to manually provide `private_vpc_connection_peering` to the parallelstore module. -You can get this details from the Google Cloud Console UI in `VPC network peering` -section. Below is the example of using existing network and creating parallelstore. -If existing network is not configured with private-service-access, you can follow -[Configure private service access](https://cloud.google.com/vpc/docs/configure-private-services-access) -to set it up. - -```yaml - - id: network - source: modules/network/pre-existing-vpc - settings: - network_name: // Add network name - subnetwork_name: // Add subnetwork name - - - id: parallelstore - source: modules/file-system/parallelstore - use: [network] - settings: - private_vpc_connection_peering: # will look like "servicenetworking.googleapis.com" -``` - -### Import data from GCS bucket - -You can import data from your GCS bucket to parallelstore instance. Important to -note that data may not be available to the instance immediately. This depends on -latency and size of data. Below is the example of importing data from bucket. - -```yaml - - id: parallelstore - source: modules/file-system/parallelstore - use: [network] - settings: - import_gcs_bucket_uri: gs://gcs-bucket/folder-path - import_destination_path: /gcs/import/ -``` - -Here you can replace `import_gcs_bucket_uri` with the uri of sub folder within GCS -bucket and `import_destination_path` with local directory within parallelstore -instance. - -### Additional configuration for DAOS agent and dfuse -Use `daos_agent_config` to provide additional configuration for `daos_agent`, for example: - -```yaml -- id: parallelstorefs - source: modules/file-system/pre-existing-network-storage - settings: - daos_agent_config: | - credential_config: - cache_expiration: 1m -``` - -Use `dfuse_environment` to provide additional environment variables for `dfuse` process, for example: - -```yaml -- id: parallelstorefs - source: modules/file-system/parallelstore - settings: - dfuse_environment: - D_LOG_FILE: /tmp/client.log - D_APPEND_PID_TO_LOG: 1 - D_LOG_MASK: debug -``` - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.13 | -| [google](#requirement\_google) | >= 6.13.0 | -| [null](#requirement\_null) | ~> 3.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.13.0 | -| [null](#provider\_null) | ~> 3.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_parallelstore_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/parallelstore_instance) | resource | -| [null_resource.hydration](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [daos\_agent\_config](#input\_daos\_agent\_config) | Additional configuration to be added to daos\_config.yml | `string` | `""` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment. | `string` | n/a | yes | -| [dfuse\_environment](#input\_dfuse\_environment) | Additional environment variables for DFuse process | `map(string)` | `{}` | no | -| [directory\_stripe](#input\_directory\_stripe) | The parallelstore stripe level for directories. | `string` | `null` | no | -| [file\_stripe](#input\_file\_stripe) | The parallelstore stripe level for files. | `string` | `null` | no | -| [import\_destination\_path](#input\_import\_destination\_path) | The name of local path to import data on parallelstore instance from GCS bucket. | `string` | `null` | no | -| [import\_gcs\_bucket\_uri](#input\_import\_gcs\_bucket\_uri) | The name of the GCS bucket to import data from to parallelstore. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to parallel store instance. | `map(string)` | `{}` | no | -| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/parallelstore"` | no | -| [mount\_options](#input\_mount\_options) | Options describing various aspects of the parallelstore instance. | `string` | `"disable-wb-cache,thread-count=16,eq-count=8"` | no | -| [name](#input\_name) | Name of parallelstore instance. | `string` | `null` | no | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to which the instance is connected given in the format:
`projects//global/networks/`" | `string` | n/a | yes | -| [private\_vpc\_connection\_peering](#input\_private\_vpc\_connection\_peering) | The name of the VPC Network peering connection.
If using new VPC, please use community/modules/network/private-service-access to create private-service-access and
If using existing VPC with private-service-access enabled, set this manually." | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [size\_gb](#input\_size\_gb) | Storage size of the parallelstore instance in GB. | `number` | `12000` | no | -| [zone](#input\_zone) | Location for parallelstore instance. | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [instructions](#output\_instructions) | Instructions to monitor import-data operation from GCS bucket to parallelstore. | -| [network\_storage](#output\_network\_storage) | Describes a parallelstore instance. | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/main.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/main.tf deleted file mode 100644 index acc2a0551e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/main.tf +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "parallelstore", ghpc_role = "file-system" }) -} - -locals { - fs_type = "daos" - server_ip = "" - remote_mount = "" - id = var.name != null ? var.name : "${var.deployment_name}-${random_id.resource_name_suffix.hex}" - access_points = jsonencode(google_parallelstore_instance.instance.access_points) - destination_path = var.import_destination_path == null ? "/" : var.import_destination_path - - client_install_runner = { - "type" = "shell" - "source" = "${path.module}/scripts/install-daos-client.sh" - "destination" = "install_daos_client.sh" - } - - mount_runner = { - "type" = "shell" - "content" = templatefile("${path.module}/templates/mount-daos.sh.tftpl", { - access_points = local.access_points - daos_agent_config = var.daos_agent_config - dfuse_environment = var.dfuse_environment - local_mount = var.local_mount - mount_options = join(" ", [for opt in split(",", var.mount_options) : "--${opt}"]) - }) - "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" - } -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_parallelstore_instance" "instance" { - project = var.project_id - instance_id = local.id - location = var.zone - capacity_gib = var.size_gb - network = var.network_id - file_stripe_level = var.file_stripe - directory_stripe_level = var.directory_stripe - - labels = local.labels - - depends_on = [var.private_vpc_connection_peering] -} - -resource "null_resource" "hydration" { - count = var.import_gcs_bucket_uri != null ? 1 : 0 - - depends_on = [resource.google_parallelstore_instance.instance] - provisioner "local-exec" { - command = "curl -X POST -H \"Content-Type: application/json\" -H \"Authorization: Bearer $(gcloud auth print-access-token)\" -d '{\"source_gcs_bucket\": {\"uri\":\"${var.import_gcs_bucket_uri}\"}, \"destination_parallelstore\": {\"path\":\"${local.destination_path}\"}}' https://parallelstore.googleapis.com/v1beta/projects/${var.project_id}/locations/${var.zone}/instances/${local.id}:importData" - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/metadata.yaml deleted file mode 100644 index c0994d15bb..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - parallelstore.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/outputs.tf deleted file mode 100644 index f6e817ac8a..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/outputs.tf +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - operation_instructions = <<-EOT - Data is being imported from GCS bucket to parallelstore instance. It may - not be available immediately. - EOT -} - -output "network_storage" { - description = "Describes a parallelstore instance." - value = { - server_ip = local.server_ip - remote_mount = local.remote_mount - local_mount = var.local_mount - fs_type = local.fs_type - mount_options = var.mount_options - client_install_runner = local.client_install_runner - mount_runner = local.mount_runner - } - - precondition { - condition = var.import_gcs_bucket_uri != null || var.import_destination_path == null - error_message = <<-EOD - Please specify import_gcs_bucket_uri to import data to parallelstore instance. - EOD - } -} - -output "instructions" { - description = "Instructions to monitor import-data operation from GCS bucket to parallelstore." - value = var.import_gcs_bucket_uri != null ? local.operation_instructions : "Data is not imported from GCS bucket." -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh deleted file mode 100644 index e96eadb56a..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/scripts/install-daos-client.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') - -if ! { - { [[ "${OS_ID}" = "rocky" ]] || [[ "${OS_ID}" = "rhel" ]]; } && { [[ "${OS_VERSION_MAJOR}" = "8" ]] || [[ "${OS_VERSION_MAJOR}" = "9" ]]; } || - { [[ "${OS_ID}" = "ubuntu" ]] && [[ "${OS_VERSION}" = "22.04" ]]; } || - { [[ "${OS_ID}" = "debian" ]] && [[ "${OS_VERSION_MAJOR}" = "12" ]]; } -}; then - echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." - exit 1 -fi - -if [ -x /bin/daos ]; then - echo "DAOS already installed" - daos version -else - # Install the DAOS client library - # The following commands should be executed on each client vm. - ## For Rocky linux 8 / RedHat 8. - if [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; then - # 1) Add the Parallelstore package repository - cat >/etc/yum.repos.d/parallelstore-v2-6-el"${OS_VERSION_MAJOR}".repo <<-EOF - [parallelstore-v2-6-el${OS_VERSION_MAJOR}] - name=Parallelstore EL${OS_VERSION_MAJOR} v2.6 - baseurl=https://us-central1-yum.pkg.dev/projects/parallelstore-packages/v2-6-el${OS_VERSION_MAJOR} - enabled=1 - repo_gpgcheck=0 - gpgcheck=0 - EOF - - ## TODO: Remove disable automatic update script after issue is fixed. - if [ -x /usr/bin/google_disable_automatic_updates ]; then - /usr/bin/google_disable_automatic_updates - fi - dnf clean all - dnf makecache - - # 2) Install daos-client - dnf install -y epel-release # needed for capstone - dnf install -y daos-client - - # 3) Upgrade libfabric - dnf upgrade -y libfabric - - # For Ubuntu 22.04 and debian 12, - elif [[ "${OS_ID}" = "ubuntu" ]] || [[ "${OS_ID}" = "debian" ]]; then - # shellcheck disable=SC2034 - DEBIAN_FRONTEND=noninteractive - - # 1) Add the Parallelstore package repository - curl -o /etc/apt/trusted.gpg.d/us-central1-apt.pkg.dev.asc https://us-central1-apt.pkg.dev/doc/repo-signing-key.gpg - echo "deb https://us-central1-apt.pkg.dev/projects/parallelstore-packages v2-6-deb main" >/etc/apt/sources.list.d/artifact-registry.list - - apt-get update - - # 2) Install daos-client - apt-get install -y daos-client - - # 3) Create daos_agent.service (comes pre-installed with RedHat) - if ! getent passwd daos_agent >/dev/null 2>&1; then - useradd daos_agent - fi - cat >/etc/systemd/system/daos_agent.service <<-EOF - [Unit] - Description=DAOS Agent - StartLimitIntervalSec=60 - Wants=network-online.target - After=network-online.target - - [Service] - Type=notify - User=daos_agent - Group=daos_agent - RuntimeDirectory=daos_agent - RuntimeDirectoryMode=0755 - ExecStart=/usr/bin/daos_agent -o /etc/daos/daos_agent.yml - StandardOutput=journal - StandardError=journal - Restart=always - RestartSec=10 - LimitMEMLOCK=infinity - LimitCORE=infinity - StartLimitBurst=5 - - [Install] - WantedBy=multi-user.target - EOF - else - echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." - exit 1 - fi -fi - -exit 0 diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl deleted file mode 100644 index c6f5d53660..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/templates/mount-daos.sh.tftpl +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') - -if ! { - { [[ "$${OS_ID}" = "rocky" ]] || [[ "$${OS_ID}" = "rhel" ]]; } && { [[ "$${OS_VERSION_MAJOR}" = "8" ]] || [[ "$${OS_VERSION_MAJOR}" = "9" ]]; } || - { [[ "$${OS_ID}" = "ubuntu" ]] && [[ "$${OS_VERSION}" = "22.04" ]]; } || - { [[ "$${OS_ID}" = "debian" ]] && [[ "$${OS_VERSION_MAJOR}" = "12" ]]; } -}; then - echo "Unsupported operating system $${OS_ID} $${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." - exit 1 - -fi - -# Edit agent config -daos_config=/etc/daos/daos_agent.yml - -# rewrite $daos_config from scratch -mv $${daos_config} $${daos_config}.orig - -exclude_fabric_ifaces="" -# Get names of network interfaces not in first PCI slot -# The first PCI slot is a standard network adapter while remaining interfaces -# are typically network cards dedicated to GPU or workload communication -if [[ "$${OS_ID}" == "debian" ]] || [[ "$${OS_ID}" = "ubuntu" ]]; then - extra_interfaces=$(find /sys/class/net/ -not -name 'enp0s*' -regextype posix-extended -regex '.*/enp[0-9]+s.*' -printf '"%f"\n' | paste -s -d ',') -elif [[ "$${OS_ID}" = "rocky" ]] || [[ "$${OS_ID}" = "rhel" ]]; then - extra_interfaces=$(find /sys/class/net/ -not -name eth0 -regextype posix-extended -regex '.*/eth[0-9]+' -printf '"%f"\n' | paste -s -d ',') -fi - -cat > $daos_config </etc/systemd/system/"$${service_name}" </global/networks/`" - EOT - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "import_gcs_bucket_uri" { - description = "The name of the GCS bucket to import data from to parallelstore." - type = string - default = null -} - -variable "import_destination_path" { - description = "The name of local path to import data on parallelstore instance from GCS bucket." - type = string - default = null -} - -variable "file_stripe" { - description = "The parallelstore stripe level for files." - type = string - default = null - validation { - condition = var.file_stripe == null ? true : contains([ - "FILE_STRIPE_LEVEL_UNSPECIFIED", - "FILE_STRIPE_LEVEL_MIN", - "FILE_STRIPE_LEVEL_BALANCED", - "FILE_STRIPE_LEVEL_MAX", - ], var.file_stripe) - error_message = "var.file_stripe must be set to \"FILE_STRIPE_LEVEL_UNSPECIFIED\", \"FILE_STRIPE_LEVEL_MIN\", \"FILE_STRIPE_LEVEL_BALANCED\", or \"FILE_STRIPE_LEVEL_MAX\"" - } -} - -variable "directory_stripe" { - description = "The parallelstore stripe level for directories." - type = string - default = null - validation { - condition = var.directory_stripe == null ? true : contains([ - "DIRECTORY_STRIPE_LEVEL_UNSPECIFIED", - "DIRECTORY_STRIPE_LEVEL_MIN", - "DIRECTORY_STRIPE_LEVEL_BALANCED", - "DIRECTORY_STRIPE_LEVEL_MAX", - ], var.directory_stripe) - error_message = "var.directory_stripe must be set to \"DIRECTORY_STRIPE_LEVEL_UNSPECIFIED\", \"DIRECTORY_STRIPE_LEVEL_MIN\", \"DIRECTORY_STRIPE_LEVEL_BALANCED\", or \"DIRECTORY_STRIPE_LEVEL_MAX\"" - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/versions.tf deleted file mode 100644 index 174b5281e4..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/parallelstore/versions.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.13" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 6.13.0" - } - - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/README.md b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/README.md deleted file mode 100644 index 1968fe4c84..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/README.md +++ /dev/null @@ -1,192 +0,0 @@ -## Description - -This module defines a file-system that already exists (i.e. it does not create -a new file system) in a way that can be shared with other modules. This allows -a compute VM to mount a filesystem that is not part of the current deployment -group. - -The pre-existing network storage can be referenced in the same way as any Cluster -Toolkit supported file-system such as [filestore](../filestore/README.md). - -For more information on network storage options in the Cluster Toolkit, see -the extended [Network Storage documentation](../../../../docs/network_storage.md). - -### Example - -```yaml -- id: homefs - source: modules/file-system/pre-existing-network-storage - settings: - server_ip: ## Set server IP here ## - remote_mount: nfsshare - local_mount: /home - fs_type: nfs -``` - -This creates a pre-existing-network-storage module in terraform at the -provided IP in `server_ip` of type nfs that will be mounted at `/home`. Note -that the `server_ip` must be known before deployment. - -The following is an example of using `pre-existing-network-storage` with a GCS -bucket: - -```yaml -- id: data-bucket - source: modules/file-system/pre-existing-network-storage - settings: - remote_mount: my-bucket-name - local_mount: /data - fs_type: gcsfuse - mount_options: defaults,_netdev,implicit_dirs -``` - -The `implicit_dirs` mount option allows object paths to be treated as if they -were directories. This is important when working with files that were created by -another source, but there may have performance impacts. The `_netdev` mount option -denotes that the storage device requires network access. - -The following is an example of using `pre-existing-network-storage` with the `lustre` -filesystem: - -```yaml -- id: lustrefs - source: modules/file-system/pre-existing-network-storage - settings: - fs_type: lustre - server_ip: 192.168.227.11@tcp - local_mount: /scratch - remote_mount: /exacloud -``` - -Note the use of the MGS NID (Network ID) in the `server_ip` field - in -particular, note the `@tcp` suffix. - -The following is an example of using `pre-existing-network-storage` with the -`managed_lustre` filesystem: - -```yaml -- id: lustrefs - source: modules/file-system/pre-existing-network-storage - settings: - fs_type: managed_lustre - server_ip: 192.168.227.11@tcp - local_mount: /scratch - remote_mount: /mg_lustre -``` - -This is similar to the `lustre` filesystem, with the exception that it connects -with a managed Lustre instance hosted by GCP. Currently only Rocky 8 and -Ubuntu 20.04 and Ubuntu 22.04 are supported. - -The following is an example of using `pre-existing-network-storage` with the `daos` -filesystem. In order to use existing `parallelstore` instance, `fs_type` needs to be -explicitly mentioned in blueprint. The `remote_mount` option refers to `access_points` -for `parallelstore` instance. - -```yaml -- id: parallelstorefs - source: modules/file-system/pre-existing-network-storage - settings: - fs_type: daos - remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" - mount_options: disable-wb-cache,thread-count=16,eq-count=8 -``` - -Parallelstore supports additional options for its mountpoints under `parallelstore_options` setting. -Use `daos_agent_config` to provide additional configuration for `daos_agent`, for example: - -```yaml -- id: parallelstorefs - source: modules/file-system/pre-existing-network-storage - settings: - fs_type: daos - remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" - mount_options: disable-wb-cache,thread-count=16,eq-count=8 - parallelstore_options: - daos_agent_config: | - credential_config: - cache_expiration: 1m -``` - -Use `dfuse_environment` to provide additional environment variables for `dfuse` process, for example: - -```yaml -- id: parallelstorefs - source: modules/file-system/pre-existing-network-storage - settings: - fs_type: daos - remote_mount: "[10.246.99.2,10.246.99.3,10.246.99.4]" - mount_options: disable-wb-cache,thread-count=16,eq-count=8 - parallelstore_options: - dfuse_environment: - D_LOG_FILE: /tmp/client.log - D_APPEND_PID_TO_LOG: 1 - D_LOG_MASK: debug -``` - -### Mounting - -For the `fs_type` listed below, this module will provide `client_install_runner` -and `mount_runner` outputs. These can be used to create a startup script to -mount the network storage system. - -Supported `fs_type`: - -- nfs -- lustre -- managed_lustre -- gcsfuse -- daos - -[scripts/mount.sh](./scripts/mount.sh) is used as the contents of -`mount_runner`. This script will update `/etc/fstab` and mount the network -storage. This script will fail if the specified `local_mount` is already being -used by another entry in `/etc/fstab`. - -Both of these steps are automatically handled with the use of the `use` command -in a selection of Cluster Toolkit modules. See the [compatibility matrix][matrix] in -the network storage doc for a complete list of supported modules. - -[matrix]: ../../../docs/network_storage.md#compatibility-matrix - -## License - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [fs\_type](#input\_fs\_type) | Type of file system to be mounted (e.g., nfs, lustre) | `string` | `"nfs"` | no | -| [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | -| [managed\_lustre\_options](#input\_managed\_lustre\_options) | Managed Lustre specific options:
gke\_support\_enabled (bool, default = false)
Note: gke\_support\_enabled does not work with Slurm, the Slurm image must be built with
the correct compatibility. | 
object({
    gke_support_enabled = optional(bool, false)
  })
| `{}` | no | -| [mount\_options](#input\_mount\_options) | Options describing various aspects of the file system. Consider adding setting to 'defaults,\_netdev,implicit\_dirs' when using gcsfuse. | `string` | `"defaults,_netdev"` | no | -| [parallelstore\_options](#input\_parallelstore\_options) | Parallelstore specific options | 
object({
    daos_agent_config = optional(string, "")
    dfuse_environment = optional(map(string), {})
  })
| `{}` | no | -| [remote\_mount](#input\_remote\_mount) | Remote FS name or export. This is the exported directory for nfs, fs name for lustre, and bucket name (without gs://) for gcsfuse. | `string` | n/a | yes | -| [server\_ip](#input\_server\_ip) | The device name as supplied to fs-tab, excluding remote fs-name(for nfs, that is the server IP, for lustre [:]). This can be omitted for gcsfuse. | `string` | `""` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [client\_install\_runner](#output\_client\_install\_runner) | Runner that performs client installation needed to use file system. | -| [mount\_runner](#output\_mount\_runner) | Runner that mounts the file system. | -| [network\_storage](#output\_network\_storage) | Describes a remote network storage to be mounted by fs-tab. | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml deleted file mode 100644 index 641832182d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/metadata.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: [] diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf deleted file mode 100644 index 203b6dfdac..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/outputs.tf +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "network_storage" { - description = "Describes a remote network storage to be mounted by fs-tab." - value = { - server_ip = var.server_ip - remote_mount = local.remote_mount - local_mount = var.local_mount - fs_type = local.fs_type - mount_options = var.mount_options - client_install_runner = local.client_install_runner - mount_runner = local.mount_runner - } -} - -locals { - # Update remote mount to include a slash if the fs_type requires one to exist - remote_mount_with_slash = length(regexall("^/.*", var.remote_mount)) > 0 ? ( - var.remote_mount - ) : format("/%s", var.remote_mount) - remote_mount = contains(local.mount_vanilla_supported_fstype, local.fs_type) ? ( - local.remote_mount_with_slash - ) : var.remote_mount - - ml_gke_support_enabled = coalesce(try(var.managed_lustre_options.gke_support_enabled, false), false) - - # Collapse fs_type lustre and managed lustre for most uses, only needs to be - # different for client installation - fs_type = strcontains(var.fs_type, "lustre") ? "lustre" : var.fs_type - - # Client Install - ddn_lustre_client_install_script = templatefile( - "${path.module}/templates/ddn_exascaler_luster_client_install.tftpl", - { - server_ip = split("@", var.server_ip)[0] - remote_mount = local.remote_mount - local_mount = var.local_mount - } - ) - managed_lustre_client_install_script = file("${path.module}/scripts/install-managed-lustre-client.sh") - nfs_client_install_script = file("${path.module}/scripts/install-nfs-client.sh") - gcs_fuse_install_script = file("${path.module}/scripts/install-gcs-fuse.sh") - daos_client_install_script = file("${path.module}/scripts/install-daos-client.sh") - - install_scripts = { - "lustre" = local.ddn_lustre_client_install_script - "managed_lustre" = local.managed_lustre_client_install_script - "nfs" = local.nfs_client_install_script - "gcsfuse" = local.gcs_fuse_install_script - "daos" = local.daos_client_install_script - } - - client_install_runner = { - "type" = "shell" - "content" = lookup(local.install_scripts, var.fs_type, "echo 'skipping: client_install_runner not yet supported for ${var.fs_type}'") - "destination" = "install_filesystem_client${replace(var.local_mount, "/", "_")}.sh" - "args" = local.ml_gke_support_enabled ? "1" : "" - } - - mount_vanilla_supported_fstype = ["lustre", "nfs"] - mount_runner_vanilla = { - "type" = "shell" - "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" - "args" = "\"${var.server_ip}\" \"${local.remote_mount}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${var.mount_options}\"" - "content" = ( - contains(local.mount_vanilla_supported_fstype, local.fs_type) ? - file("${path.module}/scripts/mount.sh") : - "echo 'skipping: mount_runner not yet supported for ${var.fs_type}'" - ) - } - gcsbucket = trimprefix(var.remote_mount, "gs://") - mount_runner_gcsfuse = { - "type" = "shell" - "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" - "args" = "\"not-used\" \"${local.gcsbucket}\" \"${var.local_mount}\" \"${local.fs_type}\" \"${var.mount_options}\"" - "content" = file("${path.module}/scripts/mount.sh") - } - - mount_runner_daos = { - "type" = "shell" - "content" = templatefile("${path.module}/templates/mount-daos.sh.tftpl", { - access_points = var.remote_mount - daos_agent_config = var.parallelstore_options.daos_agent_config - dfuse_environment = var.parallelstore_options.dfuse_environment - local_mount = var.local_mount - # avoid passing "--" as mount option to dfuse - mount_options = length(var.mount_options) == 0 ? "" : join(" ", [for opt in split(",", var.mount_options) : "--${opt}"]) - }) - "destination" = "mount_filesystem${replace(var.local_mount, "/", "_")}.sh" - } - - mount_scripts = { - "lustre" = local.mount_runner_vanilla - "nfs" = local.mount_runner_vanilla - "gcsfuse" = local.mount_runner_gcsfuse - "daos" = local.mount_runner_daos - } - - mount_runner = lookup(local.mount_scripts, local.fs_type, local.mount_runner_vanilla) -} - -output "client_install_runner" { - description = "Runner that performs client installation needed to use file system." - value = local.client_install_runner -} - -output "mount_runner" { - description = "Runner that mounts the file system." - value = local.mount_runner -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh deleted file mode 100644 index e96eadb56a..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-daos-client.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -OS_ID=$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g') -OS_VERSION_MAJOR=$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//') - -if ! { - { [[ "${OS_ID}" = "rocky" ]] || [[ "${OS_ID}" = "rhel" ]]; } && { [[ "${OS_VERSION_MAJOR}" = "8" ]] || [[ "${OS_VERSION_MAJOR}" = "9" ]]; } || - { [[ "${OS_ID}" = "ubuntu" ]] && [[ "${OS_VERSION}" = "22.04" ]]; } || - { [[ "${OS_ID}" = "debian" ]] && [[ "${OS_VERSION_MAJOR}" = "12" ]]; } -}; then - echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." - exit 1 -fi - -if [ -x /bin/daos ]; then - echo "DAOS already installed" - daos version -else - # Install the DAOS client library - # The following commands should be executed on each client vm. - ## For Rocky linux 8 / RedHat 8. - if [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; then - # 1) Add the Parallelstore package repository - cat >/etc/yum.repos.d/parallelstore-v2-6-el"${OS_VERSION_MAJOR}".repo <<-EOF - [parallelstore-v2-6-el${OS_VERSION_MAJOR}] - name=Parallelstore EL${OS_VERSION_MAJOR} v2.6 - baseurl=https://us-central1-yum.pkg.dev/projects/parallelstore-packages/v2-6-el${OS_VERSION_MAJOR} - enabled=1 - repo_gpgcheck=0 - gpgcheck=0 - EOF - - ## TODO: Remove disable automatic update script after issue is fixed. - if [ -x /usr/bin/google_disable_automatic_updates ]; then - /usr/bin/google_disable_automatic_updates - fi - dnf clean all - dnf makecache - - # 2) Install daos-client - dnf install -y epel-release # needed for capstone - dnf install -y daos-client - - # 3) Upgrade libfabric - dnf upgrade -y libfabric - - # For Ubuntu 22.04 and debian 12, - elif [[ "${OS_ID}" = "ubuntu" ]] || [[ "${OS_ID}" = "debian" ]]; then - # shellcheck disable=SC2034 - DEBIAN_FRONTEND=noninteractive - - # 1) Add the Parallelstore package repository - curl -o /etc/apt/trusted.gpg.d/us-central1-apt.pkg.dev.asc https://us-central1-apt.pkg.dev/doc/repo-signing-key.gpg - echo "deb https://us-central1-apt.pkg.dev/projects/parallelstore-packages v2-6-deb main" >/etc/apt/sources.list.d/artifact-registry.list - - apt-get update - - # 2) Install daos-client - apt-get install -y daos-client - - # 3) Create daos_agent.service (comes pre-installed with RedHat) - if ! getent passwd daos_agent >/dev/null 2>&1; then - useradd daos_agent - fi - cat >/etc/systemd/system/daos_agent.service <<-EOF - [Unit] - Description=DAOS Agent - StartLimitIntervalSec=60 - Wants=network-online.target - After=network-online.target - - [Service] - Type=notify - User=daos_agent - Group=daos_agent - RuntimeDirectory=daos_agent - RuntimeDirectoryMode=0755 - ExecStart=/usr/bin/daos_agent -o /etc/daos/daos_agent.yml - StandardOutput=journal - StandardError=journal - Restart=always - RestartSec=10 - LimitMEMLOCK=infinity - LimitCORE=infinity - StartLimitBurst=5 - - [Install] - WantedBy=multi-user.target - EOF - else - echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. This script only supports Rocky Linux 8, Redhat 8, Redhat 9, Ubuntu 22.04, and Debian 12." - exit 1 - fi -fi - -exit 0 diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh deleted file mode 100644 index f8a990260b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-gcs-fuse.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -if [ ! "$(which gcsfuse)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ]; then - tee /etc/yum.repos.d/gcsfuse.repo >/dev/null <>/etc/modprobe.d/lnet.conf - fi -fi - -if grep -q lustre /proc/filesystems; then - echo "Skipping managed lustre client install as it is already supported" - exit 0 -fi - -# Get distro information -. /etc/os-release -DIST="NA" -if [[ $NAME == *"Ubuntu"* ]]; then - if [[ $VERSION_ID == "20.04" || $VERSION_ID == "22.04" ]]; then - DIST="Ubuntu" - fi -elif [[ $NAME == *"Rocky"* ]]; then - if [[ $VERSION_ID == "8"* ]]; then - DIST="Rocky" - fi -fi - -if [[ ${DIST} == "Ubuntu" ]]; then - KEY_LOC=/etc/apt/keyrings - KEY_NAME=gcp-ar-repo.gpg - # Download new repo key - mkdir -p "${KEY_LOC}" - wget -O - https://us-apt.pkg.dev/doc/repo-signing-key.gpg 2>/dev/null | gpg --dearmor - | tee "${KEY_LOC}/${KEY_NAME}" >/dev/null - - # Set up apt repo - echo "deb [ signed-by=${KEY_LOC}/${KEY_NAME} ] https://us-apt.pkg.dev/projects/lustre-client-binaries lustre-client-ubuntu-${UBUNTU_CODENAME} main" | tee -a /etc/apt/sources.list.d/artifact-registry.list - - # Install modules - apt update - apt install -y "lustre-client-modules-$(uname -r)" lustre-client-utils || (echo "Error finding Lustre module packages, Lustre package may not exist for this kernel version" && exit 1) -elif [[ ${DIST} == "Rocky" ]]; then - # Set up yum repo - touch /etc/yum.repos.d/artifact-registry.repo - tee -a /etc/yum.repos.d/artifact-registry.repo <<-EOF - [lustre-client-rocky-8] - name=lustre-client-rocky-8 - baseurl=https://us-yum.pkg.dev/projects/lustre-client-binaries/lustre-client-rocky-8 - enabled=1 - repo_gpgcheck=0 - gpgcheck=0 - EOF - # Install modules - yum makecache - yum --enablerepo=lustre-client-rocky-8 install -y kmod-lustre-client lustre-client -fi - -if [[ $DIST != "NA" ]]; then - # Load the new lustre client module - modprobe lustre -fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh deleted file mode 100644 index 9f842c5d7c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/install-nfs-client.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ! "$(which mount.nfs)" ]; then - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || - [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - major_version=$(rpm -E "%{rhel}") - enable_repo="" - if [ "${major_version}" -eq "7" ]; then - enable_repo="base,epel" - elif [ "${major_version}" -eq "8" ] || [ "${major_version}" -eq "9" ]; then - enable_repo="baseos" - else - echo "Unsupported version of centos/RHEL/Rocky" - return 1 - fi - yum install --disablerepo="*" --enablerepo=${enable_repo} -y nfs-utils - elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get -y install nfs-common - else - echo 'Unsuported distribution' - return 1 - fi -fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh deleted file mode 100644 index e2509fb4a1..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/scripts/mount.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -SERVER_IP=$1 -REMOTE_MOUNT=$2 -LOCAL_MOUNT=$3 -FS_TYPE=$4 -MOUNT_OPTIONS=$5 - -[[ -z "${MOUNT_OPTIONS}" ]] && POPULATED_MOUNT_OPTIONS="defaults" || POPULATED_MOUNT_OPTIONS="${MOUNT_OPTIONS}" - -if [ "${FS_TYPE}" = "gcsfuse" ]; then - FS_SPEC="${REMOTE_MOUNT}" -else - FS_SPEC="${SERVER_IP}:${REMOTE_MOUNT}" -fi - -SAME_LOCAL_IDENTIFIER="^[^#].*[[:space:]]${LOCAL_MOUNT}" -EXACT_MATCH_IDENTIFIER="${FS_SPEC}[[:space:]]${LOCAL_MOUNT}[[:space:]]${FS_TYPE}[[:space:]]${POPULATED_MOUNT_OPTIONS}[[:space:]]0[[:space:]]0" - -grep -q "${SAME_LOCAL_IDENTIFIER}" /etc/fstab && SAME_LOCAL_IN_FSTAB=true || SAME_LOCAL_IN_FSTAB=false -grep -q "${EXACT_MATCH_IDENTIFIER}" /etc/fstab && EXACT_IN_FSTAB=true || EXACT_IN_FSTAB=false -findmnt --source "${SERVER_IP}":"${REMOTE_MOUNT}" --target "${LOCAL_MOUNT}" &>/dev/null && EXACT_MOUNTED=true || EXACT_MOUNTED=false - -# Do nothing and success if exact entry is already in fstab and mounted -if [ "$EXACT_IN_FSTAB" = true ] && [ "${EXACT_MOUNTED}" = true ]; then - echo "Skipping mounting source: ${FS_SPEC}, already mounted to target:${LOCAL_MOUNT}" - exit 0 -fi - -# Fail if previous fstab entry is using same local mount -if [ "$SAME_LOCAL_IN_FSTAB" = true ] && [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Mounting failed as local mount: ${LOCAL_MOUNT} was already in use in fstab" - exit 1 -fi - -# Add to fstab if entry is not already there -if [ "${EXACT_IN_FSTAB}" = false ]; then - echo "Adding ${FS_SPEC} -> ${LOCAL_MOUNT} to /etc/fstab" - echo "${FS_SPEC} ${LOCAL_MOUNT} ${FS_TYPE} ${POPULATED_MOUNT_OPTIONS} 0 0" >>/etc/fstab -fi - -# Mount from fstab -echo "Mounting --target ${LOCAL_MOUNT} from fstab" -mkdir -p "${LOCAL_MOUNT}" -mount --target "${LOCAL_MOUNT}" diff --git a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl b/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl deleted file mode 100644 index f5f0291e85..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/file-system/pre-existing-network-storage/templates/ddn_exascaler_luster_client_install.tftpl +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/sh - -# Copyright 2022 DataDirect Networks -# Modifications Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Prior Art: https://github.com/DDNStorage/exascaler-cloud-terraform/blob/78deadbb2c1fa7e4603cf9605b0f7d1782117954/gcp/templates/client-script.tftpl - -# install new EXAScaler Cloud clients: -# all instances must be in the same zone -# and connected to the same network and subnet -# to set up EXAScaler Cloud filesystem on a new client instance, -# run the following commands on the client with root privileges: -set -e -if [[ ! -z $(cat /proc/filesystems | grep lustre) ]]; then - echo "Skipping lustre client install as it is already supported" - exit 0 -fi - -cat >/etc/esc-client.conf< $daos_config </etc/systemd/system/"$${service_name}" < -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string
    count = number
    gpu_driver_installation_config = optional(object({
      gpu_driver_version = string
    }), { gpu_driver_version = "DEFAULT" })
    gpu_partition_size = optional(string)
    gpu_sharing_config = optional(object({
      gpu_sharing_strategy       = string
      max_shared_clients_per_gpu = number
    }))
  }))
| `[]` | no | -| [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [guest\_accelerator](#output\_guest\_accelerator) | Sanitized list of the type and count of accelerator cards attached to the instance. | -| [machine\_type\_guest\_accelerator](#output\_machine\_type\_guest\_accelerator) | List of the type and count of accelerator cards attached to the specified machine type. | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/internal/gpu-definition/main.tf b/vdi-test-scott/primary/modules/embedded/modules/internal/gpu-definition/main.tf deleted file mode 100644 index d85a9f8eb6..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/internal/gpu-definition/main.tf +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "machine_type" { - description = "Machine type to use for the instance creation" - type = string -} - -variable "guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the instance." - type = list(object({ - type = string - count = number - gpu_driver_installation_config = optional(object({ - gpu_driver_version = string - }), { gpu_driver_version = "DEFAULT" }) - gpu_partition_size = optional(string) - gpu_sharing_config = optional(object({ - gpu_sharing_strategy = string - max_shared_clients_per_gpu = number - })) - })) - default = [] - nullable = false -} - -locals { - # example state; terraform will ignore diffs if last element of URL matches - # guest_accelerator = [ - # { - # count = 1 - # type = "https://www.googleapis.com/compute/beta/projects/PROJECT/zones/ZONE/acceleratorTypes/nvidia-tesla-a100" - # }, - # ] - accelerator_machines = { - "a2-highgpu-1g" = { type = "nvidia-tesla-a100", count = 1 }, - "a2-highgpu-2g" = { type = "nvidia-tesla-a100", count = 2 }, - "a2-highgpu-4g" = { type = "nvidia-tesla-a100", count = 4 }, - "a2-highgpu-8g" = { type = "nvidia-tesla-a100", count = 8 }, - "a2-megagpu-16g" = { type = "nvidia-tesla-a100", count = 16 }, - "a2-ultragpu-1g" = { type = "nvidia-a100-80gb", count = 1 }, - "a2-ultragpu-2g" = { type = "nvidia-a100-80gb", count = 2 }, - "a2-ultragpu-4g" = { type = "nvidia-a100-80gb", count = 4 }, - "a2-ultragpu-8g" = { type = "nvidia-a100-80gb", count = 8 }, - "a3-highgpu-1g" = { type = "nvidia-h100-80gb", count = 1 }, - "a3-highgpu-2g" = { type = "nvidia-h100-80gb", count = 2 }, - "a3-highgpu-4g" = { type = "nvidia-h100-80gb", count = 4 }, - "a3-highgpu-8g" = { type = "nvidia-h100-80gb", count = 8 }, - "a3-megagpu-8g" = { type = "nvidia-h100-mega-80gb", count = 8 }, - "a3-ultragpu-8g" = { type = "nvidia-h200-141gb", count = 8 }, - "a4-highgpu-8g-lowmem" = { type = "nvidia-b200", count = 8 }, - "a4-highgpu-8g" = { type = "nvidia-b200", count = 8 }, - "g2-standard-4" = { type = "nvidia-l4", count = 1 }, - "g2-standard-8" = { type = "nvidia-l4", count = 1 }, - "g2-standard-12" = { type = "nvidia-l4", count = 1 }, - "g2-standard-16" = { type = "nvidia-l4", count = 1 }, - "g2-standard-24" = { type = "nvidia-l4", count = 2 }, - "g2-standard-32" = { type = "nvidia-l4", count = 1 }, - "g2-standard-48" = { type = "nvidia-l4", count = 4 }, - "g2-standard-96" = { type = "nvidia-l4", count = 8 }, - } - generated_guest_accelerator = try([local.accelerator_machines[var.machine_type]], []) - - # Select in priority order: - # (1) var.guest_accelerator if not empty - # (2) local.generated_guest_accelerator if not empty - # (3) default to empty list if both are empty - guest_accelerator = try(coalescelist(var.guest_accelerator, local.generated_guest_accelerator), []) -} - -output "guest_accelerator" { - description = "Sanitized list of the type and count of accelerator cards attached to the instance." - value = local.guest_accelerator -} - -output "machine_type_guest_accelerator" { - description = "List of the type and count of accelerator cards attached to the specified machine type." - value = local.generated_guest_accelerator -} - -terraform { - required_version = ">= 1.3" -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/README.md b/vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/README.md deleted file mode 100644 index 8aa9270a0a..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/README.md +++ /dev/null @@ -1,54 +0,0 @@ - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.0 | -| [google-beta](#requirement\_google-beta) | >= 6.0.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google-beta](#provider\_google-beta) | >= 6.0.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_compute_network_attachment.self](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_network_attachment) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [connection\_preference](#input\_connection\_preference) | The connection preference of service attachment. | `string` | `"ACCEPT_AUTOMATIC"` | no | -| [name](#input\_name) | Name of the resource. Provided by the client when the resource is created | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | The ID of the project in which the resource belongs. | `string` | n/a | yes | -| [region](#input\_region) | Region where the network attachment resides | `string` | n/a | yes | -| [subnetwork\_self\_links](#input\_subnetwork\_self\_links) | An array of selfLinks of subnets to use for endpoints in the producers that connect to this network attachment. | `list(string)` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [self\_link](#output\_self\_link) | Server-defined URL for the resource. | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/main.tf b/vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/main.tf deleted file mode 100644 index bbbece7085..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/main.tf +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - - -variable "connection_preference" { - type = string - description = "The connection preference of service attachment." - default = "ACCEPT_AUTOMATIC" -} - -variable "subnetwork_self_links" { - type = list(string) - description = " An array of selfLinks of subnets to use for endpoints in the producers that connect to this network attachment." -} - -variable "name" { - type = string - description = "Name of the resource. Provided by the client when the resource is created" -} - -variable "project_id" { - type = string - description = "The ID of the project in which the resource belongs." -} - -variable "region" { - type = string - description = "Region where the network attachment resides" -} - - -resource "google_compute_network_attachment" "self" { - provider = google-beta - - project = var.project_id - region = var.region - name = var.name - connection_preference = var.connection_preference - subnetworks = var.subnetwork_self_links -} - - -output "self_link" { - value = google_compute_network_attachment.self.self_link - description = "Server-defined URL for the resource." -} - -terraform { - required_version = ">= 0.15.0" - - required_providers { - google-beta = { - source = "hashicorp/google-beta" - version = ">= 6.0.0" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/metadata.yaml deleted file mode 100644 index e80fc96b9c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/internal/network-attachment/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/README.md b/vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/README.md deleted file mode 100644 index aefac9d187..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/README.md +++ /dev/null @@ -1,56 +0,0 @@ - -Copyright 2025 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.0 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_network_peering.peering](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_network_peering) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [export\_custom\_routes](#input\_export\_custom\_routes) | (Optional) Whether to export the custom routes to the peer network. Defaults to false. | `bool` | `null` | no | -| [import\_custom\_routes](#input\_import\_custom\_routes) | (Optional) Whether to import the custom routes from the peer network. Defaults to false. | `bool` | `null` | no | -| [import\_subnet\_routes\_with\_public\_ip](#input\_import\_subnet\_routes\_with\_public\_ip) | (Optional) Whether subnet routes with public IP range are imported. | `bool` | `null` | no | -| [name](#input\_name) | Name of the peering. | `string` | n/a | yes | -| [network\_self\_link](#input\_network\_self\_link) | The primary network of the peering. | `string` | n/a | yes | -| [peer\_network\_self\_link](#input\_peer\_network\_self\_link) | The peer network in the peering. The peer network may belong to a different project. | `string` | n/a | yes | -| [stack\_type](#input\_stack\_type) | (Optional) Which IP version(s) of traffic and routes are allowed to be imported or exported between peer networks. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [peering\_name](#output\_peering\_name) | Name of the peering. | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/main.tf b/vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/main.tf deleted file mode 100644 index 386fa9377b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/main.tf +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "name" { - type = string - description = "Name of the peering." -} - -variable "network_self_link" { - type = string - description = "The primary network of the peering." -} - -variable "peer_network_self_link" { - type = string - description = "The peer network in the peering. The peer network may belong to a different project." -} - -variable "export_custom_routes" { - type = bool - description = "(Optional) Whether to export the custom routes to the peer network. Defaults to false." - default = null -} - -variable "import_custom_routes" { - type = bool - description = "(Optional) Whether to import the custom routes from the peer network. Defaults to false." - default = null -} - -variable "import_subnet_routes_with_public_ip" { - type = bool - description = "(Optional) Whether subnet routes with public IP range are imported. " - default = null -} - -variable "stack_type" { - type = string - description = "(Optional) Which IP version(s) of traffic and routes are allowed to be imported or exported between peer networks. " - default = null -} - -resource "google_compute_network_peering" "peering" { - name = var.name - network = var.network_self_link - peer_network = var.peer_network_self_link - export_custom_routes = var.export_custom_routes - import_custom_routes = var.import_custom_routes - import_subnet_routes_with_public_ip = var.import_subnet_routes_with_public_ip - stack_type = var.stack_type -} - -output "peering_name" { - value = google_compute_network_peering.peering.name - description = "Name of the peering." -} - -terraform { - required_version = ">= 0.15.0" - - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/metadata.yaml deleted file mode 100644 index e80fc96b9c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/internal/vpc_peering/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/README.md b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/README.md deleted file mode 100644 index 432442bf61..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/README.md +++ /dev/null @@ -1,171 +0,0 @@ -## Description - -This module simplifies the following functionality: - -* Applying Kubernetes manifests to GKE clusters: It provides flexible options for specifying manifests, allowing you to either directly embed them as strings content or reference them from URLs, files, templates, or entire .yaml and .tftpl files in directories. -* Deploying commonly used infrastructure like [Kueue](https://kueue.sigs.k8s.io/docs/) or [Jobset](https://jobset.sigs.k8s.io/docs/). - -> Note: Kueue can work with a variety of frameworks out of the box, find them [here](https://kueue.sigs.k8s.io/docs/tasks/run/) - -### Explanation - -* **Manifest:** - * **Raw String:** Specify manifests directly within the module configuration using the `content: manifest_body` format. - * **File/Template/Directory Reference:** Set `source` to the path to: - * A single URL to a manifest file. Ex.: `https://github.com/.../myrepo/manifest.yaml`. - * A single local YAML manifest file (`.yaml`). Ex.: `./manifest.yaml`. - * A template file (`.tftpl`) to generate a manifest. Ex.: `./template.yaml.tftpl`. You can pass the variables to format the template file in `template_vars`. - * A directory containing multiple YAML or template files. Ex: `./manifests/`. You can pass the variables to format the template files in `template_vars`. - -#### Manifest Example - -```yaml -- id: existing-gke-cluster - source: modules/scheduler/pre-existing-gke-cluster - settings: - project_id: $(vars.project_id) - cluster_name: my-gke-cluster - region: us-central1 - -- id: kubectl-apply - source: modules/management/kubectl-apply - use: [existing-gke-cluster] - settings: - - content: | - apiVersion: v1 - kind: Namespace - metadata: - name: my-namespace - - source: "https://github.com/kubernetes-sigs/jobset/releases/download/v0.6.0/manifests.yaml" - - source: $(ghpc_stage("manifests/configmap1.yaml")) - - source: $(ghpc_stage("manifests/configmap2.yaml.tftpl")) - template_vars: {name: "dev-config", public: "false"} - - source: $(ghpc_stage("manifests"))/ - template_vars: {name: "dev-config", public: "false"} -``` - -#### Pre-build infrastructure Example - -```yaml - - id: workload_component_install - source: modules/management/kubectl-apply - use: [gke_cluster] - settings: - kueue: - install: true - config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml")) - jobset: - install: true -``` - -The `config_path` field in `kueue` installation accepts a template file, too. You will need to provide variables for the template using `config_template_vars` field. - -```yaml - - id: workload_component_install - source: modules/management/kubectl-apply - use: [gke_cluster] - settings: - kueue: - install: true - config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml.tftpl")) - config_template_vars: {name: "dev-config", public: "false"} - jobset: - install: true -``` - -You can specify a particular kueue version that you would like to use using the `version` flag. By default, we recommend customers to [use v0.10.0](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/management/kubectl-apply/variables.tf#L68). You can find the list of supported kueue versions [here](https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/modules/management/kubectl-apply/variables.tf#L18). - -```yaml - - id: workload_component_install - source: modules/management/kubectl-apply - use: [gke_cluster] - settings: - kueue: - install: true - version: v0.10.0 - config_path: $(ghpc_stage("manifests/user-provided-kueue-config.yaml.tftpl")) - config_template_vars: {name: "dev-config", public: "false"} - jobset: - install: true -``` - -> **_NOTE:_** -> -> The `project_id` and `region` settings would be inferred from the deployment variables of the same name, but they are included here for clarity. -> -> Terraform may apply resources in parallel, leading to potential dependency issues. If a resource's dependencies aren't ready, it will be applied again up to 15 times. - -## License - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | > 5.0 | -| [helm](#requirement\_helm) | ~> 2.17 | -| [kubectl](#requirement\_kubectl) | >= 1.7.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | > 5.0 | -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [configure\_kueue](#module\_configure\_kueue) | ./kubectl | n/a | -| [install\_gib](#module\_install\_gib) | ./kubectl | n/a | -| [install\_gpu\_operator](#module\_install\_gpu\_operator) | ./helm_install | n/a | -| [install\_jobset](#module\_install\_jobset) | ./kubectl | n/a | -| [install\_kueue](#module\_install\_kueue) | ./kubectl | n/a | -| [install\_nvidia\_dra\_driver](#module\_install\_nvidia\_dra\_driver) | ./helm_install | n/a | -| [kubectl\_apply\_manifests](#module\_kubectl\_apply\_manifests) | ./kubectl | n/a | - -## Resources - -| Name | Type | -|------|------| -| [terraform_data.gib_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [terraform_data.initial_gib_version](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [terraform_data.jobset_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [terraform_data.kueue_validations](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | -| [google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [apply\_manifests](#input\_apply\_manifests) | A list of manifests to apply to GKE cluster using kubectl. For more details see [kubectl module's inputs](kubectl/README.md). | 
list(object({
    enable            = optional(bool, true)
    content           = optional(string, null)
    source            = optional(string, null)
    template_vars     = optional(map(any), null)
    server_side_apply = optional(bool, false)
    wait_for_rollout  = optional(bool, true)
  }))
| `[]` | no | -| [cluster\_id](#input\_cluster\_id) | An identifier for the gke cluster resource with format projects//locations//clusters/. | `string` | n/a | yes | -| [gib](#input\_gib) | Install the NCCL gIB plugin | 
object({
    install = bool
    path    = string
    template_vars = object({
      image   = optional(string, "us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib")
      version = string
      node_affinity = optional(any, {
        requiredDuringSchedulingIgnoredDuringExecution = {
          nodeSelectorTerms = [{
            matchExpressions = [{
              key      = "cloud.google.com/gke-gpu",
              operator = "In",
              values   = ["true"]
            }]
          }]
        }
      })
      accelerator_count = number
    })
  })
| 
{
  "install": false,
  "path": "",
  "template_vars": {
    "accelerator_count": 0,
    "version": ""
  }
}
| no | -| [gke\_cluster\_exists](#input\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | `bool` | `false` | no | -| [gpu\_operator](#input\_gpu\_operator) | Install [GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) which uses the [Kubernetes operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) to automate the management of all NVIDIA software components needed to provision GPU. | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0")
  })
| `{}` | no | -| [jobset](#input\_jobset) | Install [Jobset](https://github.com/kubernetes-sigs/jobset) which manages a group of K8s [jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) as a unit. | 
object({
    install = optional(bool, false)
    version = optional(string, "v0.7.2")
  })
| `{}` | no | -| [kueue](#input\_kueue) | Install and configure [Kueue](https://kueue.sigs.k8s.io/docs/overview/) workload scheduler. A configuration yaml/template file can be provided with config\_path to be applied right after kueue installation. If a template file provided, its variables can be set to config\_template\_vars. | 
object({
    install              = optional(bool, false)
    version              = optional(string, "v0.12.2")
    config_path          = optional(string, null)
    config_template_vars = optional(map(any), null)
  })
| `{}` | no | -| [nvidia\_dra\_driver](#input\_nvidia\_dra\_driver) | Installs [Nvidia DRA driver](https://github.com/NVIDIA/k8s-dra-driver-gpu) which supports Dynamic Resource Allocation for NVIDIA GPUs in Kubernetes | 
object({
    install = optional(bool, false)
    version = optional(string, "v25.3.0-rc.3")
  })
| `{}` | no | -| [project\_id](#input\_project\_id) | The project ID that hosts the gke cluster. | `string` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/README.md b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/README.md deleted file mode 100644 index 1957899617..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/README.md +++ /dev/null @@ -1,64 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [helm](#requirement\_helm) | ~> 2.17 | - -## Providers - -| Name | Version | -|------|---------| -| [helm](#provider\_helm) | ~> 2.17 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [helm_release.apply_chart](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [atomic](#input\_atomic) | If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used. | `bool` | `false` | no | -| [chart\_name](#input\_chart\_name) | Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL). | `string` | n/a | yes | -| [chart\_repository](#input\_chart\_repository) | URL of the Helm chart repository. Set to null or omit if 'chart\_name' is a path or URL. | `string` | `null` | no | -| [chart\_version](#input\_chart\_version) | Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true). | `string` | `null` | no | -| [cleanup\_on\_fail](#input\_cleanup\_on\_fail) | Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail'). | `bool` | `false` | no | -| [create\_namespace](#input\_create\_namespace) | Set to true to create the namespace if it does not exist ('helm install --create-namespace'). | `bool` | `true` | no | -| [dependency\_update](#input\_dependency\_update) | Run 'helm dependency update' before installing the chart (useful if chart\_name is a local path to an unpacked chart with dependencies). | `bool` | `false` | no | -| [description](#input\_description) | Set an optional description for the Helm release. | `string` | `null` | no | -| [devel](#input\_devel) | Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart\_version' is set, this is ignored. | `bool` | `false` | no | -| [disable\_crd\_hooks](#input\_disable\_crd\_hooks) | Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook'). | `bool` | `false` | no | -| [disable\_openapi\_validation](#input\_disable\_openapi\_validation) | If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation'). | `bool` | `false` | no | -| [disable\_webhooks](#input\_disable\_webhooks) | Prevent hooks from running ('helm install --no-hooks'). | `bool` | `false` | no | -| [force\_update](#input\_force\_update) | Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution. | `bool` | `false` | no | -| [keyring](#input\_keyring) | Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true. | `string` | `null` | no | -| [lint](#input\_lint) | Run the helm chart linter during the plan ('helm lint'). | `bool` | `false` | no | -| [max\_history](#input\_max\_history) | Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit. | `number` | `null` | no | -| [namespace](#input\_namespace) | Kubernetes namespace to install the Helm release into. | `string` | `"default"` | no | -| [pass\_credentials](#input\_pass\_credentials) | Pass credentials to all domains ('helm install --pass-credentials'). Use with caution. | `bool` | `false` | no | -| [postrender](#input\_postrender) | Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary\_path' attribute. | 
object({
    binary_path = string # Path to the post-renderer executable
  })
| `null` | no | -| [recreate\_pods](#input\_recreate\_pods) | Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself. | `bool` | `false` | no | -| [release\_name](#input\_release\_name) | Name of the Helm release. | `string` | n/a | yes | -| [render\_subchart\_notes](#input\_render\_subchart\_notes) | If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes'). | `bool` | `false` | no | -| [reset\_values](#input\_reset\_values) | When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values'). | `bool` | `false` | no | -| [reuse\_values](#input\_reuse\_values) | When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset\_values' is specified, this is ignored. | `bool` | `false` | no | -| [set\_values](#input\_set\_values) | List of objects defining values to set ('helm install --set'). | 
list(object({
    name  = string                     # Path to the value (e.g., 'service.type', 'replicaCount')
    value = string                     # The value to set
    type  = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file')
  }))
| `[]` | no | -| [skip\_crds](#input\_skip\_crds) | If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present. | `bool` | `false` | no | -| [timeout](#input\_timeout) | Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout'). | `number` | `300` | no | -| [values\_yaml](#input\_values\_yaml) | List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile(). | `list(string)` | `[]` | no | -| [verify](#input\_verify) | Verify the package before installing it ('helm install --verify'). | `bool` | `false` | no | -| [wait](#input\_wait) | Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait'). | `bool` | `true` | no | -| [wait\_for\_jobs](#input\_wait\_for\_jobs) | If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs'). | `bool` | `false` | no | - -## Outputs - -No outputs. - diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf deleted file mode 100644 index 8cc09bd3e2..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/main.tf +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resource "helm_release" "apply_chart" { - # Required Identification - name = var.release_name - chart = var.chart_name - - # Chart Source & Version - repository = var.chart_repository - version = var.chart_version - devel = var.devel - - # Target Namespace - namespace = var.namespace - create_namespace = var.create_namespace - - # Values Configuration - values = var.values_yaml - - dynamic "set" { - for_each = var.set_values - content { - name = set.value.name - value = set.value.value - type = set.value.type - } - } - - # Installation/Upgrade Behavior - description = var.description - atomic = var.atomic - cleanup_on_fail = var.cleanup_on_fail - dependency_update = var.dependency_update - disable_crd_hooks = var.disable_crd_hooks - disable_openapi_validation = var.disable_openapi_validation - disable_webhooks = var.disable_webhooks - force_update = var.force_update - lint = var.lint - max_history = var.max_history - recreate_pods = var.recreate_pods # Note: Deprecated in Helm CLI - render_subchart_notes = var.render_subchart_notes - reset_values = var.reset_values - reuse_values = var.reuse_values - skip_crds = var.skip_crds - timeout = var.timeout - wait = var.wait - wait_for_jobs = var.wait_for_jobs - - # Verification & Credentials - keyring = var.keyring - pass_credentials = var.pass_credentials - verify = var.verify - - # Post Rendering - dynamic "postrender" { - # Only include the block if var.postrender is not null - for_each = var.postrender == null ? [] : [var.postrender] - content { - binary_path = postrender.value.binary_path - } - } - - # Lifecycle block (optional - generally avoid complex lifecycle in generic modules) - # lifecycle { - # ignore_changes = [] - # } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml deleted file mode 100644 index 17bedb471b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf deleted file mode 100644 index 04e8e214fc..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/variables.tf +++ /dev/null @@ -1,212 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Description: Input variables for the generic Helm release module. - -# --- Required --- -variable "release_name" { - description = "Name of the Helm release." - type = string -} - -variable "chart_name" { - description = "Name of the Helm chart (can be a chart reference, path to a packaged chart, path to an unpacked chart directory, or a URL)." - type = string -} - -# --- Chart Location & Version --- -variable "chart_repository" { - description = "URL of the Helm chart repository. Set to null or omit if 'chart_name' is a path or URL." - type = string - default = null -} - -variable "chart_version" { - description = "Version of the Helm chart to install. If omitted, the latest version will be selected (unless 'devel' is true)." - type = string - default = null -} - -variable "devel" { - description = "Use development versions, too ('helm install --devel'). Equivalent to version '>0.0.0-0'. If 'chart_version' is set, this is ignored." - type = bool - default = false -} - -# --- Namespace --- -variable "namespace" { - description = "Kubernetes namespace to install the Helm release into." - type = string - default = "default" -} - -variable "create_namespace" { - description = "Set to true to create the namespace if it does not exist ('helm install --create-namespace')." - type = bool - default = true # Common convenience setting -} - -# --- Values Customization --- -variable "values_yaml" { - description = "List of YAML strings or paths to YAML files containing chart values ('helm install -f'). Can use file() or templatefile()." - type = list(string) - default = [] -} - -variable "set_values" { - description = "List of objects defining values to set ('helm install --set')." - type = list(object({ - name = string # Path to the value (e.g., 'service.type', 'replicaCount') - value = string # The value to set - type = optional(string, "string") # Type of value ('string', 'json', 'yaml', 'file') - })) - default = [] -} - -# --- Installation/Upgrade Behavior --- -variable "description" { - description = "Set an optional description for the Helm release." - type = string - default = null -} - -variable "atomic" { - description = "If set, the installation process purges chart on failure ('helm install --atomic'). The --wait flag will be set automatically if atomic is used." - type = bool - default = false -} - -variable "wait" { - description = "Will wait until all resources are in a ready state before marking the release as successful ('helm install --wait')." - type = bool - default = true # Often a good default for dependencies -} - -variable "wait_for_jobs" { - description = "If 'wait' is enabled, will wait until all Jobs have been completed before marking the release as successful ('helm install --wait-for-jobs')." - type = bool - default = false # Helm CLI default is false -} - -variable "timeout" { - description = "Time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks) ('helm install --timeout')." - type = number - default = 300 # 5 minutes (Helm CLI default) -} - -variable "cleanup_on_fail" { - description = "Allow deletion of new resources created in this upgrade when the upgrade fails ('helm upgrade --cleanup-on-fail')." - type = bool - default = false -} - -variable "dependency_update" { - description = "Run 'helm dependency update' before installing the chart (useful if chart_name is a local path to an unpacked chart with dependencies)." - type = bool - default = false -} - -variable "disable_crd_hooks" { - description = "Prevent CRD hooks from running, but run other hooks ('helm install --no-crd-hook')." - type = bool - default = false -} - -variable "disable_openapi_validation" { - description = "If set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema ('helm install --disable-openapi-validation')." - type = bool - default = false -} - -variable "disable_webhooks" { - description = "Prevent hooks from running ('helm install --no-hooks')." - type = bool - default = false -} - -variable "force_update" { - description = "Force resource update through delete/recreate if needed ('helm upgrade --force'). Use with caution." - type = bool - default = false -} - -variable "lint" { - description = "Run the helm chart linter during the plan ('helm lint')." - type = bool - default = false -} - -variable "max_history" { - description = "Limit the maximum number of revisions saved per release ('helm upgrade --history-max'). 0 for no limit." - type = number - default = null # Terraform provider defaults to Helm's default (usually 10) -} - -variable "recreate_pods" { - description = "Perform pods restart for the resource if applicable ('helm upgrade --recreate-pods'). Note: This flag is deprecated in Helm CLI v3 itself." - type = bool - default = false -} - -variable "render_subchart_notes" { - description = "If set, render subchart notes along with the parent chart's notes ('helm install --render-subchart-notes')." - type = bool - default = false -} - -variable "reset_values" { - description = "When upgrading, reset the values to the ones built into the chart ('helm upgrade --reset-values')." - type = bool - default = false -} - -variable "reuse_values" { - description = "When upgrading, reuse the last release's values and merge in any overrides ('helm upgrade --reuse-values'). If 'reset_values' is specified, this is ignored." - type = bool - default = false # Helm CLI default is false -} - -variable "skip_crds" { - description = "If set, no CRDs will be installed ('helm install --skip-crds'). By default, CRDs are installed if not present." - type = bool - default = false -} - -# --- Verification & Credentials --- -variable "keyring" { - description = "Location of public keys used for verification ('helm install --keyring'). Used if 'verify' is true." - type = string - default = null # Defaults to Helm's default keyring location -} - -variable "pass_credentials" { - description = "Pass credentials to all domains ('helm install --pass-credentials'). Use with caution." - type = bool - default = false -} - -variable "verify" { - description = "Verify the package before installing it ('helm install --verify')." - type = bool - default = false -} - -# --- Advanced Rendering --- -variable "postrender" { - description = "Configuration for a post-rendering executable ('helm install --post-renderer'). Should be an object with 'binary_path' attribute." - type = object({ - binary_path = string # Path to the post-renderer executable - }) - default = null # Disabled by default -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf deleted file mode 100644 index 09d912e2c9..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/helm_install/versions.tf +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_providers { - helm = { - source = "hashicorp/helm" - version = "~> 2.17" - } - } - - required_version = ">= 1.3" -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/README.md b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/README.md deleted file mode 100644 index a57421b81f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/README.md +++ /dev/null @@ -1,57 +0,0 @@ - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [http](#requirement\_http) | ~> 3.0 | -| [kubectl](#requirement\_kubectl) | >= 1.7.0 | - -## Providers - -| Name | Version | -|------|---------| -| [http](#provider\_http) | ~> 3.0 | -| [kubectl](#provider\_kubectl) | >= 1.7.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [kubectl_manifest.apply_doc](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [http_http.yaml_content](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source | -| [kubectl_path_documents.templates](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/path_documents) | data source | -| [kubectl_path_documents.yamls](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/data-sources/path_documents) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [content](#input\_content) | The YAML body to apply to gke cluster. | `string` | `null` | no | -| [server\_side\_apply](#input\_server\_side\_apply) | Allow using kubectl server-side apply method. | `bool` | `false` | no | -| [source\_path](#input\_source\_path) | The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file. | `string` | `null` | no | -| [template\_vars](#input\_template\_vars) | The values to populate template file(s) with. | `any` | `null` | no | -| [wait\_for\_rollout](#input\_wait\_for\_rollout) | Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details. | `bool` | `true` | no | - -## Outputs - -No outputs. - diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf deleted file mode 100644 index c2abe3ae94..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/main.tf +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - yaml_separator = "\n---" - - content_yaml_body = var.content - - null_safe_source = coalesce(var.source_path, " ") - - url = startswith(local.null_safe_source, "http://") || startswith(local.null_safe_source, "https://") ? var.source_path : null - url_content = local.url != null ? data.http.yaml_content[0].response_body : null - - yaml_file = local.url == null && length(regexall("\\.yaml(_.*)?$", lower(local.null_safe_source))) == 1 ? abspath(var.source_path) : null - yaml_file_content = local.yaml_file != null ? file(local.yaml_file) : null - - template_file = local.url == null && length(regexall("\\.tftpl(_.*)?$", lower(local.null_safe_source))) == 1 ? abspath(var.source_path) : null - template_file_content = local.template_file != null ? templatefile(local.template_file, var.template_vars) : null - - yaml_body = coalesce(local.content_yaml_body, local.url_content, local.yaml_file_content, local.template_file_content, " ") - yaml_body_docs = [for doc in split(local.yaml_separator, local.yaml_body) : trimspace(doc) if length(trimspace(doc)) > 0] # Split yaml to single docs because the kubectl provider only supports single resource - - directory = length(local.yaml_body_docs) == 0 && endswith(local.null_safe_source, "/") ? abspath(var.source_path) : null - - docs_list = concat(try(local.yaml_body_docs, []), try(data.kubectl_path_documents.yamls[0].documents, []), try(data.kubectl_path_documents.templates[0].documents, [])) - docs_map = tomap({ - for index, doc in local.docs_list : index => doc - }) -} - -data "http" "yaml_content" { - count = local.url != null ? 1 : 0 - url = local.url -} - -data "kubectl_path_documents" "yamls" { - count = local.directory != null ? 1 : 0 - pattern = "${local.directory}/*.yaml" -} - -data "kubectl_path_documents" "templates" { - count = local.directory != null ? 1 : 0 - pattern = "${local.directory}/*.tftpl" - vars = var.template_vars -} - -resource "kubectl_manifest" "apply_doc" { - for_each = local.docs_map - yaml_body = each.value - server_side_apply = var.server_side_apply - wait_for_rollout = var.wait_for_rollout -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml deleted file mode 100644 index 17bedb471b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf deleted file mode 100644 index 54a996b8f3..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/variables.tf +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "content" { - description = "The YAML body to apply to gke cluster." - type = string - default = null -} - -variable "source_path" { - description = "The source for manifest(s) to apply to gke cluster. Acceptable sources are a local yaml or template (.tftpl) file path, a directory (ends with '/') containing yaml or template files, and a url for a yaml file." - type = string - default = null -} - -variable "template_vars" { - description = "The values to populate template file(s) with." - type = any - default = null -} - -variable "server_side_apply" { - description = "Allow using kubectl server-side apply method." - type = bool - default = false -} - -variable "wait_for_rollout" { - description = "Wait or not for Deployments and APIService to complete rollout. See [kubectl wait](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_wait/) for more details." - type = bool - default = true -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf deleted file mode 100644 index 0275fb90a7..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/kubectl/versions.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - kubectl = { - source = "gavinbunney/kubectl" - version = ">= 1.7.0" - } - http = { - source = "hashicorp/http" - version = "~> 3.0" - } - } - - required_version = ">= 1.3" -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/main.tf b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/main.tf deleted file mode 100644 index db85c65f32..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/main.tf +++ /dev/null @@ -1,234 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - cluster_id_parts = split("/", var.cluster_id) - cluster_name = local.cluster_id_parts[5] - cluster_location = local.cluster_id_parts[3] - project_id = var.project_id != null ? var.project_id : local.cluster_id_parts[1] - - apply_manifests_map = tomap({ - for index, manifest in var.apply_manifests : index => manifest - if manifest.enable - }) - - install_kueue = try(var.kueue.install, false) - install_jobset = try(var.jobset.install, false) - install_gpu_operator = try(var.gpu_operator.install, false) - install_nvidia_dra_driver = try(var.nvidia_dra_driver.install, false) - install_gib = try(var.gib.install, false) - kueue_install_source = format("${path.module}/manifests/kueue-%s.yaml", try(var.kueue.version, "")) - jobset_install_source = format("${path.module}/manifests/jobset-%s.yaml", try(var.jobset.version, "")) -} - -data "google_container_cluster" "gke_cluster" { - project = local.project_id - name = local.cluster_name - location = local.cluster_location -} - -data "google_client_config" "default" {} - -module "kubectl_apply_manifests" { - for_each = local.apply_manifests_map - source = "./kubectl" - depends_on = [var.gke_cluster_exists] - - content = each.value.content - source_path = each.value.source - template_vars = each.value.template_vars - server_side_apply = each.value.server_side_apply - wait_for_rollout = each.value.wait_for_rollout - - providers = { - kubectl = kubectl - } -} - -module "install_kueue" { - source = "./kubectl" - source_path = local.install_kueue ? local.kueue_install_source : null - server_side_apply = true - wait_for_rollout = true - depends_on = [var.gke_cluster_exists] - - providers = { - kubectl = kubectl - } -} - -module "configure_kueue" { - source = "./kubectl" - source_path = local.install_kueue ? try(var.kueue.config_path, "") : null - template_vars = local.install_kueue ? try(var.kueue.config_template_vars, null) : null - depends_on = [module.install_kueue] - - server_side_apply = true - wait_for_rollout = true - - providers = { - kubectl = kubectl - } -} - -module "install_jobset" { - source = "./kubectl" - source_path = local.install_jobset ? local.jobset_install_source : null - server_side_apply = true - wait_for_rollout = true - depends_on = [var.gke_cluster_exists, module.configure_kueue] - - providers = { - kubectl = kubectl - } -} - -module "install_nvidia_dra_driver" { - count = local.install_nvidia_dra_driver ? 1 : 0 - depends_on = [module.kubectl_apply_manifests, var.gke_cluster_exists, module.configure_kueue] - source = "./helm_install" - - release_name = "nvidia-dra-driver-gpu" # The release name - chart_repository = "https://helm.ngc.nvidia.com/nvidia" # The Helm repository URL for nvidia charts - chart_name = "nvidia-dra-driver-gpu" # The chart name - chart_version = var.nvidia_dra_driver.version # The chart version - namespace = "nvidia-dra-driver-gpu" # The target namespace - create_namespace = true # Equivalent to --create-namespace - - # Use the 'values' argument to pass the YAML content - # This corresponds to the -f <(cat <---. - type: boolean - subdomain: - description: |- - Subdomain is an explicit choice for a network subdomain name - When set, any replicated job in the set is added to this network. - Defaults to if not set. - type: string - type: object - x-kubernetes-validations: - - message: Value is immutable - rule: self == oldSelf - replicatedJobs: - description: ReplicatedJobs is the group of jobs that will form the set. - items: - properties: - name: - description: |- - Name is the name of the entry and will be used as a suffix - for the Job name. - type: string - replicas: - default: 1 - description: |- - Replicas is the number of jobs that will be created from this ReplicatedJob's template. - Jobs names will be in the format: -- - format: int32 - type: integer - template: - description: Template defines the template of the Job that will be created. - properties: - metadata: - description: |- - Standard object's metadata of the jobs created from this template. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - Specification of the desired behavior of the job. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - activeDeadlineSeconds: - description: |- - Specifies the duration in seconds relative to the startTime that the job - may be continuously active before the system tries to terminate it; value - must be positive integer. If a Job is suspended (at creation or through an - update), this timer will effectively be stopped and reset when the Job is - resumed again. - format: int64 - type: integer - backoffLimit: - description: |- - Specifies the number of retries before marking this job failed. - Defaults to 6 - format: int32 - type: integer - backoffLimitPerIndex: - description: |- - Specifies the limit for the number of retries within an - index before marking this index as failed. When enabled the number of - failures per index is kept in the pod's - batch.kubernetes.io/job-index-failure-count annotation. It can only - be set when Job's completionMode=Indexed, and the Pod's restart - policy is Never. The field is immutable. - This field is beta-level. It can be used when the `JobBackoffLimitPerIndex` - feature gate is enabled (enabled by default). - format: int32 - type: integer - completionMode: - description: |- - completionMode specifies how Pod completions are tracked. It can be - `NonIndexed` (default) or `Indexed`. - - - `NonIndexed` means that the Job is considered complete when there have - been .spec.completions successfully completed Pods. Each Pod completion is - homologous to each other. - - - `Indexed` means that the Pods of a - Job get an associated completion index from 0 to (.spec.completions - 1), - available in the annotation batch.kubernetes.io/job-completion-index. - The Job is considered complete when there is one successfully completed Pod - for each index. - When value is `Indexed`, .spec.completions must be specified and - `.spec.parallelism` must be less than or equal to 10^5. - In addition, The Pod name takes the form - `$(job-name)-$(index)-$(random-string)`, - the Pod hostname takes the form `$(job-name)-$(index)`. - - - More completion modes can be added in the future. - If the Job controller observes a mode that it doesn't recognize, which - is possible during upgrades due to version skew, the controller - skips updates for the Job. - type: string - completions: - description: |- - Specifies the desired number of successfully finished pods the - job should be run with. Setting to null means that the success of any - pod signals the success of all pods, and allows parallelism to have any positive - value. Setting to 1 means that parallelism is limited to 1 and the success of that - pod signals the success of the job. - More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ - format: int32 - type: integer - manualSelector: - description: |- - manualSelector controls generation of pod labels and pod selectors. - Leave `manualSelector` unset unless you are certain what you are doing. - When false or unset, the system pick labels unique to this job - and appends those labels to the pod template. When true, - the user is responsible for picking unique labels and specifying - the selector. Failure to pick a unique label may cause this - and other jobs to not function correctly. However, You may see - `manualSelector=true` in jobs that were created with the old `extensions/v1beta1` - API. - More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#specifying-your-own-pod-selector - type: boolean - maxFailedIndexes: - description: |- - Specifies the maximal number of failed indexes before marking the Job as - failed, when backoffLimitPerIndex is set. Once the number of failed - indexes exceeds this number the entire Job is marked as Failed and its - execution is terminated. When left as null the job continues execution of - all of its indexes and is marked with the `Complete` Job condition. - It can only be specified when backoffLimitPerIndex is set. - It can be null or up to completions. It is required and must be - less than or equal to 10^4 when is completions greater than 10^5. - This field is beta-level. It can be used when the `JobBackoffLimitPerIndex` - feature gate is enabled (enabled by default). - format: int32 - type: integer - parallelism: - description: |- - Specifies the maximum desired number of pods the job should - run at any given time. The actual number of pods running in steady state will - be less than this number when ((.spec.completions - .status.successful) < .spec.parallelism), - i.e. when the work left to do is less than max parallelism. - More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ - format: int32 - type: integer - podFailurePolicy: - description: |- - Specifies the policy of handling failed pods. In particular, it allows to - specify the set of actions and conditions which need to be - satisfied to take the associated action. - If empty, the default behaviour applies - the counter of failed pods, - represented by the jobs's .status.failed field, is incremented and it is - checked against the backoffLimit. This field cannot be used in combination - with restartPolicy=OnFailure. - - - This field is beta-level. It can be used when the `JobPodFailurePolicy` - feature gate is enabled (enabled by default). - properties: - rules: - description: |- - A list of pod failure policy rules. The rules are evaluated in order. - Once a rule matches a Pod failure, the remaining of the rules are ignored. - When no rule matches the Pod failure, the default handling applies - the - counter of pod failures is incremented and it is checked against - the backoffLimit. At most 20 elements are allowed. - items: - description: |- - PodFailurePolicyRule describes how a pod failure is handled when the requirements are met. - One of onExitCodes and onPodConditions, but not both, can be used in each rule. - properties: - action: - description: |- - Specifies the action taken on a pod failure when the requirements are satisfied. - Possible values are: - - - - FailJob: indicates that the pod's job is marked as Failed and all - running pods are terminated. - - FailIndex: indicates that the pod's index is marked as Failed and will - not be restarted. - This value is beta-level. It can be used when the - `JobBackoffLimitPerIndex` feature gate is enabled (enabled by default). - - Ignore: indicates that the counter towards the .backoffLimit is not - incremented and a replacement pod is created. - - Count: indicates that the pod is handled in the default way - the - counter towards the .backoffLimit is incremented. - Additional values are considered to be added in the future. Clients should - react to an unknown action by skipping the rule. - type: string - onExitCodes: - description: Represents the requirement on the container exit codes. - properties: - containerName: - description: |- - Restricts the check for exit codes to the container with the - specified name. When null, the rule applies to all containers. - When specified, it should match one the container or initContainer - names in the pod template. - type: string - operator: - description: |- - Represents the relationship between the container exit code(s) and the - specified values. Containers completed with success (exit code 0) are - excluded from the requirement check. Possible values are: - - - - In: the requirement is satisfied if at least one container exit code - (might be multiple if there are multiple containers not restricted - by the 'containerName' field) is in the set of specified values. - - NotIn: the requirement is satisfied if at least one container exit code - (might be multiple if there are multiple containers not restricted - by the 'containerName' field) is not in the set of specified values. - Additional values are considered to be added in the future. Clients should - react to an unknown operator by assuming the requirement is not satisfied. - type: string - values: - description: |- - Specifies the set of values. Each returned container exit code (might be - multiple in case of multiple containers) is checked against this set of - values with respect to the operator. The list of values must be ordered - and must not contain duplicates. Value '0' cannot be used for the In operator. - At least one element is required. At most 255 elements are allowed. - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - - values - type: object - onPodConditions: - description: |- - Represents the requirement on the pod conditions. The requirement is represented - as a list of pod condition patterns. The requirement is satisfied if at - least one pattern matches an actual pod condition. At most 20 elements are allowed. - items: - description: |- - PodFailurePolicyOnPodConditionsPattern describes a pattern for matching - an actual pod condition type. - properties: - status: - description: |- - Specifies the required Pod condition status. To match a pod condition - it is required that the specified status equals the pod condition status. - Defaults to True. - type: string - type: - description: |- - Specifies the required Pod condition type. To match a pod condition - it is required that specified type equals the pod condition type. - type: string - required: - - status - - type - type: object - type: array - x-kubernetes-list-type: atomic - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic - required: - - rules - type: object - podReplacementPolicy: - description: |- - podReplacementPolicy specifies when to create replacement Pods. - Possible values are: - - TerminatingOrFailed means that we recreate pods - when they are terminating (has a metadata.deletionTimestamp) or failed. - - Failed means to wait until a previously created Pod is fully terminated (has phase - Failed or Succeeded) before creating a replacement Pod. - - - When using podFailurePolicy, Failed is the the only allowed value. - TerminatingOrFailed and Failed are allowed values when podFailurePolicy is not in use. - This is an beta field. To use this, enable the JobPodReplacementPolicy feature toggle. - This is on by default. - type: string - selector: - description: |- - A label query over pods that should match the pod count. - Normally, the system sets this field for you. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - suspend: - description: |- - suspend specifies whether the Job controller should create Pods or not. If - a Job is created with suspend set to true, no Pods are created by the Job - controller. If a Job is suspended after creation (i.e. the flag goes from - false to true), the Job controller will delete all active Pods associated - with this Job. Users must design their workload to gracefully handle this. - Suspending a Job will reset the StartTime field of the Job, effectively - resetting the ActiveDeadlineSeconds timer too. Defaults to false. - type: boolean - template: - description: |- - Describes the pod that will be created when executing a job. - The only allowed template.spec.restartPolicy values are "Never" or "OnFailure". - More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ - properties: - metadata: - description: |- - Standard object's metadata. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - Specification of the desired behavior of the pod. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - activeDeadlineSeconds: - description: |- - Optional duration in seconds the pod may be active on the node relative to - StartTime before the system will actively try to mark it failed and kill associated containers. - Value must be a positive integer. - format: int64 - type: integer - affinity: - description: If specified, the pod's scheduling constraints - properties: - nodeAffinity: - description: Describes node affinity scheduling rules for the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node matches the corresponding matchExpressions; the - node(s) with the highest sum are the most preferred. - items: - description: |- - An empty preferred scheduling term matches all objects with implicit weight 0 - (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated with the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - x-kubernetes-map-type: atomic - weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. - items: - description: |- - A null or empty node selector term matches no objects. The requirements of - them are ANDed. - The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - x-kubernetes-map-type: atomic - type: array - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the anti-affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the anti-affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the anti-affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - Also, MatchLabelKeys cannot be set when LabelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. - Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - automountServiceAccountToken: - description: AutomountServiceAccountToken indicates whether a service account token should be automatically mounted. - type: boolean - containers: - description: |- - List of containers belonging to the pod. - Containers cannot currently be added or removed. - There must be at least one container in a Pod. - Cannot be updated. - items: - description: A single application container that you want to run within a pod. - properties: - args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment variable present in a Container. - properties: - name: - description: Name of the environment variable. Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's namespace - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - optional: - description: Specify whether the ConfigMap must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - optional: - description: Specify whether the Secret must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration that the container should sleep before being terminated. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration that the container should sleep before being terminated. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. - type: string - ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. - type: string - securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default is DefaultProcMount which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to the container. - type: string - role: - description: Role is a SELinux role label that applies to the container. - type: string - type: - description: Type is a SELinux type label that applies to the container. - type: string - user: - description: User is a SELinux user label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. - items: - description: volumeDevice describes a mapping of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. - type: string - name: - description: name must match the name of a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. - items: - description: VolumeMount describes a mounting of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - type: string - name: - description: This must match the Name of a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - dnsConfig: - description: |- - Specifies the DNS parameters of a pod. - Parameters specified here will be merged to the generated DNS - configuration based on DNSPolicy. - properties: - nameservers: - description: |- - A list of DNS name server IP addresses. - This will be appended to the base nameservers generated from DNSPolicy. - Duplicated nameservers will be removed. - items: - type: string - type: array - options: - description: |- - A list of DNS resolver options. - This will be merged with the base options generated from DNSPolicy. - Duplicated entries will be removed. Resolution options given in Options - will override those that appear in the base DNSPolicy. - items: - description: PodDNSConfigOption defines DNS resolver options of a pod. - properties: - name: - description: Required. - type: string - value: - type: string - type: object - type: array - searches: - description: |- - A list of DNS search domains for host-name lookup. - This will be appended to the base search paths generated from DNSPolicy. - Duplicated search paths will be removed. - items: - type: string - type: array - type: object - dnsPolicy: - description: |- - Set DNS policy for the pod. - Defaults to "ClusterFirst". - Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. - DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. - To have DNS options set along with hostNetwork, you have to specify DNS policy - explicitly to 'ClusterFirstWithHostNet'. - type: string - enableServiceLinks: - description: |- - EnableServiceLinks indicates whether information about services should be injected into pod's - environment variables, matching the syntax of Docker links. - Optional: Defaults to true. - type: boolean - ephemeralContainers: - description: |- - List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing - pod to perform user-initiated actions such as debugging. This list cannot be specified when - creating a pod, and it cannot be modified by updating the pod spec. In order to add an - ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. - items: - description: |- - An EphemeralContainer is a temporary container that you may add to an existing Pod for - user-initiated activities such as debugging. Ephemeral containers have no resource or - scheduling guarantees, and they will not be restarted when they exit or when a Pod is - removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the - Pod to exceed its resource allocation. - - - To add an ephemeral container, use the ephemeralcontainers subresource of an existing - Pod. Ephemeral containers may not be removed or restarted. - properties: - args: - description: |- - Arguments to the entrypoint. - The image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - command: - description: |- - Entrypoint array. Not executed within a shell. - The image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment variable present in a Container. - properties: - name: - description: Name of the environment variable. Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's namespace - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - optional: - description: Specify whether the ConfigMap must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - optional: - description: Specify whether the Secret must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: Lifecycle is not allowed for ephemeral containers. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration that the container should sleep before being terminated. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration that the container should sleep before being terminated. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: Probes are not allowed for ephemeral containers. - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the ephemeral container specified as a DNS_LABEL. - This name must be unique among all containers, init containers and ephemeral containers. - type: string - ports: - description: Ports are not allowed for ephemeral containers. - items: - description: ContainerPort represents a network port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: Probes are not allowed for ephemeral containers. - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources - already allocated to the pod. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - Restart policy for the container to manage the restart behavior of each - container within a pod. - This may only be set for init containers. You cannot set this field on - ephemeral containers. - type: string - securityContext: - description: |- - Optional: SecurityContext defines the security options the ephemeral container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default is DefaultProcMount which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to the container. - type: string - role: - description: Role is a SELinux role label that applies to the container. - type: string - type: - description: Type is a SELinux type label that applies to the container. - type: string - user: - description: User is a SELinux user label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: Probes are not allowed for ephemeral containers. - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - targetContainerName: - description: |- - If set, the name of the container from PodSpec that this ephemeral container targets. - The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. - If not set then the ephemeral container uses the namespaces configured in the Pod spec. - - - The container runtime must implement support for this feature. If the runtime does not - support namespace targeting then the result of setting this field is undefined. - type: string - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. - items: - description: volumeDevice describes a mapping of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. - type: string - name: - description: name must match the name of a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. - Cannot be updated. - items: - description: VolumeMount describes a mounting of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - type: string - name: - description: This must match the Name of a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - hostAliases: - description: |- - HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts - file if specified. This is only valid for non-hostNetwork pods. - items: - description: |- - HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the - pod's hosts file. - properties: - hostnames: - description: Hostnames for the above IP address. - items: - type: string - type: array - ip: - description: IP address of the host file entry. - type: string - type: object - type: array - hostIPC: - description: |- - Use the host's ipc namespace. - Optional: Default to false. - type: boolean - hostNetwork: - description: |- - Host networking requested for this pod. Use the host's network namespace. - If this option is set, the ports that will be used must be specified. - Default to false. - type: boolean - hostPID: - description: |- - Use the host's pid namespace. - Optional: Default to false. - type: boolean - hostUsers: - description: |- - Use the host's user namespace. - Optional: Default to true. - If set to true or not present, the pod will be run in the host user namespace, useful - for when the pod needs a feature only available to the host user namespace, such as - loading a kernel module with CAP_SYS_MODULE. - When set to false, a new userns is created for the pod. Setting false is useful for - mitigating container breakout vulnerabilities even allowing users to run their - containers as root without actually having root privileges on the host. - This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. - type: boolean - hostname: - description: |- - Specifies the hostname of the Pod - If not specified, the pod's hostname will be set to a system-defined value. - type: string - imagePullSecrets: - description: |- - ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. - If specified, these secrets will be passed to individual puller implementations for them to use. - More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod - items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - type: array - initContainers: - description: |- - List of initialization containers belonging to the pod. - Init containers are executed in order prior to containers being started. If any - init container fails, the pod is considered to have failed and is handled according - to its restartPolicy. The name for an init container or normal container must be - unique among all containers. - Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. - The resourceRequirements of an init container are taken into account during scheduling - by finding the highest request/limit for each resource type, and then using the max of - of that value or the sum of the normal containers. Limits are applied to init containers - in a similar fashion. - Init containers cannot currently be added or removed. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ - items: - description: A single application container that you want to run within a pod. - properties: - args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment variable present in a Container. - properties: - name: - description: Name of the environment variable. Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's namespace - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - optional: - description: Specify whether the ConfigMap must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - optional: - description: Specify whether the Secret must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration that the container should sleep before being terminated. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration that the container should sleep before being terminated. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. - type: string - ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. - type: string - securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default is DefaultProcMount which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to the container. - type: string - role: - description: Role is a SELinux role label that applies to the container. - type: string - type: - description: Type is a SELinux type label that applies to the container. - type: string - user: - description: User is a SELinux user label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. - items: - description: volumeDevice describes a mapping of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. - type: string - name: - description: name must match the name of a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. - items: - description: VolumeMount describes a mounting of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - type: string - name: - description: This must match the Name of a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - nodeName: - description: |- - NodeName is a request to schedule this pod onto a specific node. If it is non-empty, - the scheduler simply schedules this pod onto that node, assuming that it fits resource - requirements. - type: string - nodeSelector: - additionalProperties: - type: string - description: |- - NodeSelector is a selector which must be true for the pod to fit on a node. - Selector which must match a node's labels for the pod to be scheduled on that node. - More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - type: object - x-kubernetes-map-type: atomic - os: - description: |- - Specifies the OS of the containers in the pod. - Some pod and container fields are restricted if this is set. - - - If the OS field is set to linux, the following fields must be unset: - -securityContext.windowsOptions - - - If the OS field is set to windows, following fields must be unset: - - spec.hostPID - - spec.hostIPC - - spec.hostUsers - - spec.securityContext.seLinuxOptions - - spec.securityContext.seccompProfile - - spec.securityContext.fsGroup - - spec.securityContext.fsGroupChangePolicy - - spec.securityContext.sysctls - - spec.shareProcessNamespace - - spec.securityContext.runAsUser - - spec.securityContext.runAsGroup - - spec.securityContext.supplementalGroups - - spec.containers[*].securityContext.seLinuxOptions - - spec.containers[*].securityContext.seccompProfile - - spec.containers[*].securityContext.capabilities - - spec.containers[*].securityContext.readOnlyRootFilesystem - - spec.containers[*].securityContext.privileged - - spec.containers[*].securityContext.allowPrivilegeEscalation - - spec.containers[*].securityContext.procMount - - spec.containers[*].securityContext.runAsUser - - spec.containers[*].securityContext.runAsGroup - properties: - name: - description: |- - Name is the name of the operating system. The currently supported values are linux and windows. - Additional value may be defined in future and can be one of: - https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration - Clients should expect to handle additional values and treat unrecognized values in this field as os: null - type: string - required: - - name - type: object - overhead: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. - This field will be autopopulated at admission time by the RuntimeClass admission controller. If - the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. - The RuntimeClass admission controller will reject Pod create requests which have the overhead already - set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value - defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. - More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md - type: object - preemptionPolicy: - description: |- - PreemptionPolicy is the Policy for preempting pods with lower priority. - One of Never, PreemptLowerPriority. - Defaults to PreemptLowerPriority if unset. - type: string - priority: - description: |- - The priority value. Various system components use this field to find the - priority of the pod. When Priority Admission Controller is enabled, it - prevents users from setting this field. The admission controller populates - this field from PriorityClassName. - The higher the value, the higher the priority. - format: int32 - type: integer - priorityClassName: - description: |- - If specified, indicates the pod's priority. "system-node-critical" and - "system-cluster-critical" are two special keywords which indicate the - highest priorities with the former being the highest priority. Any other - name must be defined by creating a PriorityClass object with that name. - If not specified, the pod priority will be default or zero if there is no - default. - type: string - readinessGates: - description: |- - If specified, all readiness gates will be evaluated for pod readiness. - A pod is ready when all its containers are ready AND - all conditions specified in the readiness gates have status equal to "True" - More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates - items: - description: PodReadinessGate contains the reference to a pod condition - properties: - conditionType: - description: ConditionType refers to a condition in the pod's condition list with matching type. - type: string - required: - - conditionType - type: object - type: array - resourceClaims: - description: |- - ResourceClaims defines which ResourceClaims must be allocated - and reserved before the Pod is allowed to start. The resources - will be made available to those containers which consume them - by name. - - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - - This field is immutable. - items: - description: |- - PodResourceClaim references exactly one ResourceClaim through a ClaimSource. - It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. - Containers that need access to the ResourceClaim reference it with this name. - properties: - name: - description: |- - Name uniquely identifies this resource claim inside the pod. - This must be a DNS_LABEL. - type: string - source: - description: Source describes where to find the ResourceClaim. - properties: - resourceClaimName: - description: |- - ResourceClaimName is the name of a ResourceClaim object in the same - namespace as this pod. - type: string - resourceClaimTemplateName: - description: |- - ResourceClaimTemplateName is the name of a ResourceClaimTemplate - object in the same namespace as this pod. - - - The template will be used to create a new ResourceClaim, which will - be bound to this pod. When this pod is deleted, the ResourceClaim - will also be deleted. The pod name and resource name, along with a - generated component, will be used to form a unique name for the - ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. - - - This field is immutable and no changes will be made to the - corresponding ResourceClaim by the control plane after creating the - ResourceClaim. - type: string - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - restartPolicy: - description: |- - Restart policy for all containers within the pod. - One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. - Default to Always. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy - type: string - runtimeClassName: - description: |- - RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used - to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. - If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an - empty definition that uses the default runtime handler. - More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class - type: string - schedulerName: - description: |- - If specified, the pod will be dispatched by specified scheduler. - If not specified, the pod will be dispatched by default scheduler. - type: string - schedulingGates: - description: |- - SchedulingGates is an opaque list of values that if specified will block scheduling the pod. - If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the - scheduler will not attempt to schedule the pod. - - - SchedulingGates can only be set at pod creation time, and be removed only afterwards. - - - This is a beta feature enabled by the PodSchedulingReadiness feature gate. - items: - description: PodSchedulingGate is associated to a Pod to guard its scheduling. - properties: - name: - description: |- - Name of the scheduling gate. - Each scheduling gate must have a unique name field. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - securityContext: - description: |- - SecurityContext holds pod-level security attributes and common container settings. - Optional: Defaults to empty. See type description for default values of each field. - properties: - fsGroup: - description: |- - A special supplemental group that applies to all containers in a pod. - Some volume types allow the Kubelet to change the ownership of that volume - to be owned by the pod: - - - 1. The owning GID will be the FSGroup - 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) - 3. The permission bits are OR'd with rw-rw---- - - - If unset, the Kubelet will not modify the ownership and permissions of any volume. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - fsGroupChangePolicy: - description: |- - fsGroupChangePolicy defines behavior of changing ownership and permission of the volume - before being exposed inside Pod. This field will only apply to - volume types which support fsGroup based ownership(and permissions). - It will have no effect on ephemeral volume types such as: secret, configmaps - and emptydir. - Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. - Note that this field cannot be set when spec.os.name is windows. - type: string - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in SecurityContext. If set in - both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to the container. - type: string - role: - description: Role is a SELinux role label that applies to the container. - type: string - type: - description: Type is a SELinux type label that applies to the container. - type: string - user: - description: User is a SELinux user label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - supplementalGroups: - description: |- - A list of groups applied to the first process run in each container, in addition - to the container's primary GID, the fsGroup (if specified), and group memberships - defined in the container image for the uid of the container process. If unspecified, - no additional groups are added to any container. Note that group memberships - defined in the container image for the uid of the container process are still effective, - even if they are not included in this list. - Note that this field cannot be set when spec.os.name is windows. - items: - format: int64 - type: integer - type: array - sysctls: - description: |- - Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported - sysctls (by the container runtime) might fail to launch. - Note that this field cannot be set when spec.os.name is windows. - items: - description: Sysctl defines a kernel parameter to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options within a container's SecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - serviceAccount: - description: |- - DeprecatedServiceAccount is a depreciated alias for ServiceAccountName. - Deprecated: Use serviceAccountName instead. - type: string - serviceAccountName: - description: |- - ServiceAccountName is the name of the ServiceAccount to use to run this pod. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ - type: string - setHostnameAsFQDN: - description: |- - If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). - In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). - In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. - If a pod does not have FQDN, this has no effect. - Default to false. - type: boolean - shareProcessNamespace: - description: |- - Share a single process namespace between all of the containers in a pod. - When this is set containers will be able to view and signal processes from other containers - in the same pod, and the first process in each container will not be assigned PID 1. - HostPID and ShareProcessNamespace cannot both be set. - Optional: Default to false. - type: boolean - subdomain: - description: |- - If specified, the fully qualified Pod hostname will be "...svc.". - If not specified, the pod will not have a domainname at all. - type: string - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - If this value is nil, the default grace period will be used instead. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - Defaults to 30 seconds. - format: int64 - type: integer - tolerations: - description: If specified, the pod's tolerations. - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - topologySpreadConstraints: - description: |- - TopologySpreadConstraints describes how a group of pods ought to spread across topology - domains. Scheduler will schedule pods in a way which abides by the constraints. - All topologySpreadConstraints are ANDed. - items: - description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. - properties: - labelSelector: - description: |- - LabelSelector is used to find matching pods. - Pods that match this label selector are counted to determine the number of pods - in their corresponding topology domain. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select the pods over which - spreading will be calculated. The keys are used to lookup values from the - incoming pod labels, those key-value labels are ANDed with labelSelector - to select the group of existing pods over which spreading will be calculated - for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - MatchLabelKeys cannot be set when LabelSelector isn't set. - Keys that don't exist in the incoming pod labels will - be ignored. A null or empty list means only match against labelSelector. - - - This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxSkew: - description: |- - MaxSkew describes the degree to which pods may be unevenly distributed. - When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference - between the number of matching pods in the target topology and the global minimum. - The global minimum is the minimum number of matching pods in an eligible domain - or zero if the number of eligible domains is less than MinDomains. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 2/2/1: - In this case, the global minimum is 1. - | zone1 | zone2 | zone3 | - | P P | P P | P | - - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; - scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) - violate MaxSkew(1). - - if MaxSkew is 2, incoming pod can be scheduled onto any zone. - When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence - to topologies that satisfy it. - It's a required field. Default value is 1 and 0 is not allowed. - format: int32 - type: integer - minDomains: - description: |- - MinDomains indicates a minimum number of eligible domains. - When the number of eligible domains with matching topology keys is less than minDomains, - Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. - And when the number of eligible domains with matching topology keys equals or greater than minDomains, - this value has no effect on scheduling. - As a result, when the number of eligible domains is less than minDomains, - scheduler won't schedule more than maxSkew Pods to those domains. - If value is nil, the constraint behaves as if MinDomains is equal to 1. - Valid values are integers greater than 0. - When value is not nil, WhenUnsatisfiable must be DoNotSchedule. - - - For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same - labelSelector spread as 2/2/2: - | zone1 | zone2 | zone3 | - | P P | P P | P P | - The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. - In this situation, new pod with the same labelSelector cannot be scheduled, - because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, - it will violate MaxSkew. - - - This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). - format: int32 - type: integer - nodeAffinityPolicy: - description: |- - NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector - when calculating pod topology spread skew. Options are: - - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. - - - If this value is nil, the behavior is equivalent to the Honor policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. - type: string - nodeTaintsPolicy: - description: |- - NodeTaintsPolicy indicates how we will treat node taints when calculating - pod topology spread skew. Options are: - - Honor: nodes without taints, along with tainted nodes for which the incoming pod - has a toleration, are included. - - Ignore: node taints are ignored. All nodes are included. - - - If this value is nil, the behavior is equivalent to the Ignore policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. - type: string - topologyKey: - description: |- - TopologyKey is the key of node labels. Nodes that have a label with this key - and identical values are considered to be in the same topology. - We consider each as a "bucket", and try to put balanced number - of pods into each bucket. - We define a domain as a particular instance of a topology. - Also, we define an eligible domain as a domain whose nodes meet the requirements of - nodeAffinityPolicy and nodeTaintsPolicy. - e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. - And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. - It's a required field. - type: string - whenUnsatisfiable: - description: |- - WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy - the spread constraint. - - DoNotSchedule (default) tells the scheduler not to schedule it. - - ScheduleAnyway tells the scheduler to schedule the pod in any location, - but giving higher precedence to topologies that would help reduce the - skew. - A constraint is considered "Unsatisfiable" for an incoming pod - if and only if every possible node assignment for that pod would violate - "MaxSkew" on some topology. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 3/1/1: - | zone1 | zone2 | zone3 | - | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled - to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies - MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler - won't make it *more* imbalanced. - It's a required field. - type: string - required: - - maxSkew - - topologyKey - - whenUnsatisfiable - type: object - type: array - x-kubernetes-list-map-keys: - - topologyKey - - whenUnsatisfiable - x-kubernetes-list-type: map - volumes: - description: |- - List of volumes that can be mounted by containers belonging to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes - items: - description: Volume represents a named volume in a pod that may be accessed by any container in the pod. - properties: - awsElasticBlockStore: - description: |- - awsElasticBlockStore represents an AWS Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - TODO: how do we prevent errors in the filesystem from compromising the machine - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - format: int32 - type: integer - readOnly: - description: |- - readOnly value true will force the readOnly setting in VolumeMounts. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: boolean - volumeID: - description: |- - volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: string - required: - - volumeID - type: object - azureDisk: - description: azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. - properties: - cachingMode: - description: 'cachingMode is the Host Caching mode: None, Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data disk in the blob storage - type: string - diskURI: - description: diskURI is the URI of data disk in the blob storage - type: string - fsType: - description: |- - fsType is Filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - kind: - description: 'kind expected values are Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared' - type: string - readOnly: - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: azureFile represents an Azure File Service mount on the host and bind mount to the pod. - properties: - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret that contains Azure Storage Account Name and Key - type: string - shareName: - description: shareName is the azure share Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: cephFS represents a Ceph FS mount on the host that shares a pod's lifetime - properties: - monitors: - description: |- - monitors is Required: Monitors is a collection of Ceph monitors - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - items: - type: string - type: array - path: - description: 'path is Optional: Used as the mounted root, rather than the full Ceph tree, default is /' - type: string - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: boolean - secretFile: - description: |- - secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - secretRef: - description: |- - secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - user: - description: |- - user is optional: User is the rados user name, default is admin - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - required: - - monitors - type: object - cinder: - description: |- - cinder represents a cinder volume attached and mounted on kubelets host machine. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: boolean - secretRef: - description: |- - secretRef is optional: points to a secret object containing parameters used to connect - to OpenStack. - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - volumeID: - description: |- - volumeID used to identify the volume in cinder. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap that should populate this volume - properties: - defaultMode: - description: |- - defaultMode is optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - optional: - description: optional specify whether the ConfigMap or its keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - csi: - description: csi (Container Storage Interface) represents ephemeral storage that is handled by certain external CSI drivers (Beta feature). - properties: - driver: - description: |- - driver is the name of the CSI driver that handles this volume. - Consult with your admin for the correct name as registered in the cluster. - type: string - fsType: - description: |- - fsType to mount. Ex. "ext4", "xfs", "ntfs". - If not provided, the empty value is passed to the associated CSI driver - which will determine the default filesystem to apply. - type: string - nodePublishSecretRef: - description: |- - nodePublishSecretRef is a reference to the secret object containing - sensitive information to pass to the CSI driver to complete the CSI - NodePublishVolume and NodeUnpublishVolume calls. - This field is optional, and may be empty if no secret is required. If the - secret object contains more than one secret, all secret references are passed. - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - readOnly: - description: |- - readOnly specifies a read-only configuration for the volume. - Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: |- - volumeAttributes stores driver-specific properties that are passed to the CSI - driver. Consult your driver's documentation for supported values. - type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API about the pod that should populate this volume - properties: - defaultMode: - description: |- - Optional: mode bits to use on created files by default. Must be a - Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: Items is a list of downward API volume file - items: - description: DownwardAPIVolumeFile represents information to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.' - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - type: object - emptyDir: - description: |- - emptyDir represents a temporary directory that shares a pod's lifetime. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - properties: - medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: |- - ephemeral represents a volume that is handled by a cluster storage driver. - The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, - and deleted when the pod is removed. - - - Use this if: - a) the volume is only needed while the pod runs, - b) features of normal volumes like restoring from snapshot or capacity - tracking are needed, - c) the storage driver is specified through a storage class, and - d) the storage driver supports dynamic volume provisioning through - a PersistentVolumeClaim (see EphemeralVolumeSource for more - information on the connection between this volume type - and PersistentVolumeClaim). - - - Use PersistentVolumeClaim or one of the vendor-specific - APIs for volumes that persist for longer than the lifecycle - of an individual pod. - - - Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to - be used that way - see the documentation of the driver for - more information. - - - A pod can use both types of ephemeral volumes and - persistent volumes at the same time. - properties: - volumeClaimTemplate: - description: |- - Will be used to create a stand-alone PVC to provision the volume. - The pod in which this EphemeralVolumeSource is embedded will be the - owner of the PVC, i.e. the PVC will be deleted together with the - pod. The name of the PVC will be `-` where - `` is the name from the `PodSpec.Volumes` array - entry. Pod validation will reject the pod if the concatenated name - is not valid for a PVC (for example, too long). - - - An existing PVC with that name that is not owned by the pod - will *not* be used for the pod to avoid using an unrelated - volume by mistake. Starting the pod is then blocked until - the unrelated PVC is removed. If such a pre-created PVC is - meant to be used by the pod, the PVC has to updated with an - owner reference to the pod once the pod exists. Normally - this should not be necessary, but it may be useful when - manually reconstructing a broken cluster. - - - This field is read-only and no changes will be made by Kubernetes - to the PVC after it has been created. - - - Required, must not be nil. - properties: - metadata: - description: |- - May contain labels and annotations that will be copied into the PVC - when creating it. No other fields are allowed and will be rejected during - validation. - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - The specification for the PersistentVolumeClaim. The entire content is - copied unchanged into the PVC that gets created from this - template. The same fields as in a PersistentVolumeClaim - are also valid here. - properties: - accessModes: - description: |- - accessModes contains the desired access modes the volume should have. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 - items: - type: string - type: array - dataSource: - description: |- - dataSource field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - If the provisioner or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified data source. - When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, - and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. - If the namespace is specified, then dataSourceRef will not be copied to dataSource. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: |- - dataSourceRef specifies the object from which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a non-empty API group (non - core object) or a PersistentVolumeClaim object. - When this field is specified, volume binding will only succeed if the type of - the specified object matches some installed volume populator or dynamic - provisioner. - This field will replace the functionality of the dataSource field and as such - if both fields are non-empty, they must have the same value. For backwards - compatibility, when namespace isn't specified in dataSourceRef, - both fields (dataSource and dataSourceRef) will be set to the same - value automatically if one of them is empty and the other is non-empty. - When namespace is specified in dataSourceRef, - dataSource isn't set to the same value and must be empty. - There are three important differences between dataSource and dataSourceRef: - * While dataSource only allows two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping them), dataSourceRef - preserves all values, and generates an error if a disallowed value is - specified. - * While dataSource only allows local objects, dataSourceRef allows objects - in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. - (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - namespace: - description: |- - Namespace is the namespace of resource being referenced - Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. - (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: |- - resources represents the minimum resources the volume should have. - If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements - that are lower than previous value but must still be higher than capacity recorded in the - status field of the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - description: selector is a label query over volumes to consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: |- - storageClassName is the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 - type: string - volumeAttributesClassName: - description: |- - volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. - If specified, the CSI driver will create or update the volume with the attributes defined - in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. - If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be - set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource - exists. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#volumeattributesclass - (Alpha) Using this field requires the VolumeAttributesClass feature gate to be enabled. - type: string - volumeMode: - description: |- - volumeMode defines what type of volume is required by the claim. - Value of Filesystem is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference to the PersistentVolume backing this claim. - type: string - type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - TODO: how do we prevent errors in the filesystem from compromising the machine - type: string - lun: - description: 'lun is Optional: FC target lun number' - format: int32 - type: integer - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target worldwide names (WWNs)' - items: - type: string - type: array - wwids: - description: |- - wwids Optional: FC volume world wide identifiers (wwids) - Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. - items: - type: string - type: array - type: object - flexVolume: - description: |- - flexVolume represents a generic volume resource that is - provisioned/attached using an exec based plugin. - properties: - driver: - description: driver is the name of the driver to use for this volume. - type: string - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field holds extra command options if any.' - type: object - readOnly: - description: |- - readOnly is Optional: defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef is Optional: secretRef is reference to the secret object containing - sensitive information to pass to the plugin scripts. This may be - empty if no secret object is specified. If the secret object - contains more than one secret, all secrets are passed to the plugin - scripts. - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - required: - - driver - type: object - flocker: - description: flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running - properties: - datasetName: - description: |- - datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker - should be considered as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the dataset. This is unique identifier of a Flocker dataset - type: string - type: object - gcePersistentDisk: - description: |- - gcePersistentDisk represents a GCE Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - properties: - fsType: - description: |- - fsType is filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - TODO: how do we prevent errors in the filesystem from compromising the machine - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - format: int32 - type: integer - pdName: - description: |- - pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: boolean - required: - - pdName - type: object - gitRepo: - description: |- - gitRepo represents a git repository at a particular revision. - DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an - EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir - into the Pod's container. - properties: - directory: - description: |- - directory is the target directory name. - Must not contain or start with '..'. If '.' is supplied, the volume directory will be the - git repository. Otherwise, if specified, the volume will contain the git repository in - the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for the specified revision. - type: string - required: - - repository - type: object - glusterfs: - description: |- - glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. - More info: https://examples.k8s.io/volumes/glusterfs/README.md - properties: - endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: string - path: - description: |- - path is the Glusterfs volume path. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: string - readOnly: - description: |- - readOnly here will force the Glusterfs volume to be mounted with read-only permissions. - Defaults to false. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: boolean - required: - - endpoints - - path - type: object - hostPath: - description: |- - hostPath represents a pre-existing file or directory on the host - machine that is directly exposed to the container. This is generally - used for system agents or other privileged things that are allowed - to see the host machine. Most containers will NOT need this. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - --- - TODO(jonesdl) We need to restrict who can use host directory mounts and who can/can not - mount host directories as read/write. - properties: - path: - description: |- - path of the directory on the host. - If the path is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - type: - description: |- - type for HostPath Volume - Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - required: - - path - type: object - iscsi: - description: |- - iscsi represents an ISCSI Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md - properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether support iSCSI Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether support iSCSI Session CHAP authentication - type: boolean - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - TODO: how do we prevent errors in the filesystem from compromising the machine - type: string - initiatorName: - description: |- - initiatorName is the custom iSCSI Initiator Name. - If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface - : will be created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified Name. - type: string - iscsiInterface: - description: |- - iscsiInterface is the interface Name that uses an iSCSI transport. - Defaults to 'default' (tcp). - type: string - lun: - description: lun represents iSCSI Target Lun number. - format: int32 - type: integer - portals: - description: |- - portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - items: - type: string - type: array - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret for iSCSI target and initiator authentication - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - targetPortal: - description: |- - targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - description: |- - name of the volume. - Must be a DNS_LABEL and unique within the pod. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - nfs: - description: |- - nfs represents an NFS mount on the host that shares a pod's lifetime - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - properties: - path: - description: |- - path that is exported by the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - readOnly: - description: |- - readOnly here will force the NFS export to be mounted with read-only permissions. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: boolean - server: - description: |- - server is the hostname or IP address of the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: |- - persistentVolumeClaimVolumeSource represents a reference to a - PersistentVolumeClaim in the same namespace. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - properties: - claimName: - description: |- - claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - type: string - readOnly: - description: |- - readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - pdID: - description: pdID is the ID that identifies Photon Controller persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: portworxVolume represents a portworx volume attached and mounted on kubelets host machine - properties: - fsType: - description: |- - fSType represents the filesystem type to mount - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources secrets, configmaps, and downward API - properties: - defaultMode: - description: |- - defaultMode are the mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - sources: - description: sources is the list of volume projections - items: - description: Projection that may be projected along with other supported volume types - properties: - clusterTrustBundle: - description: |- - ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field - of ClusterTrustBundle objects in an auto-updating file. - - - Alpha, gated by the ClusterTrustBundleProjection feature gate. - - - ClusterTrustBundle objects can either be selected by name, or by the - combination of signer name and a label selector. - - - Kubelet performs aggressive normalization of the PEM contents written - into the pod filesystem. Esoteric PEM features such as inter-block - comments and block headers are stripped. Certificates are deduplicated. - The ordering of certificates within the file is arbitrary, and Kubelet - may change the order over time. - properties: - labelSelector: - description: |- - Select all ClusterTrustBundles that match this label selector. Only has - effect if signerName is set. Mutually-exclusive with name. If unset, - interpreted as "match nothing". If set but empty, interpreted as "match - everything". - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - name: - description: |- - Select a single ClusterTrustBundle by object name. Mutually-exclusive - with signerName and labelSelector. - type: string - optional: - description: |- - If true, don't block pod startup if the referenced ClusterTrustBundle(s) - aren't available. If using name, then the named ClusterTrustBundle is - allowed not to exist. If using signerName, then the combination of - signerName and labelSelector is allowed to match zero - ClusterTrustBundles. - type: boolean - path: - description: Relative path from the volume root to write the bundle. - type: string - signerName: - description: |- - Select all ClusterTrustBundles that match this signer name. - Mutually-exclusive with name. The contents of all selected - ClusterTrustBundles will be unified and deduplicated. - type: string - required: - - path - type: object - configMap: - description: configMap information about the configMap data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - optional: - description: optional specify whether the ConfigMap or its keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - downwardAPI: - description: downwardAPI information about the downwardAPI data to project - properties: - items: - description: Items is a list of DownwardAPIVolume file - items: - description: DownwardAPIVolumeFile represents information to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.' - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - type: object - secret: - description: secret information about the secret data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - optional: - description: optional field specify whether the Secret or its key must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - serviceAccountToken: - description: serviceAccountToken is information about the serviceAccountToken data to project - properties: - audience: - description: |- - audience is the intended audience of the token. A recipient of a token - must identify itself with an identifier specified in the audience of the - token, and otherwise should reject the token. The audience defaults to the - identifier of the apiserver. - type: string - expirationSeconds: - description: |- - expirationSeconds is the requested duration of validity of the service - account token. As the token approaches expiration, the kubelet volume - plugin will proactively rotate the service account token. The kubelet will - start trying to rotate the token if the token is older than 80 percent of - its time to live or if the token is older than 24 hours.Defaults to 1 hour - and must be at least 10 minutes. - format: int64 - type: integer - path: - description: |- - path is the path relative to the mount point of the file to project the - token into. - type: string - required: - - path - type: object - type: object - type: array - type: object - quobyte: - description: quobyte represents a Quobyte mount on the host that shares a pod's lifetime - properties: - group: - description: |- - group to map volume access to - Default is no group - type: string - readOnly: - description: |- - readOnly here will force the Quobyte volume to be mounted with read-only permissions. - Defaults to false. - type: boolean - registry: - description: |- - registry represents a single or multiple Quobyte Registry services - specified as a string as host:port pair (multiple entries are separated with commas) - which acts as the central registry for volumes - type: string - tenant: - description: |- - tenant owning the given Quobyte volume in the Backend - Used with dynamically provisioned Quobyte volumes, value is set by the plugin - type: string - user: - description: |- - user to map volume access to - Defaults to serivceaccount user - type: string - volume: - description: volume is a string that references an already created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: |- - rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. - More info: https://examples.k8s.io/volumes/rbd/README.md - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - TODO: how do we prevent errors in the filesystem from compromising the machine - type: string - image: - description: |- - image is the rados image name. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - keyring: - description: |- - keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - monitors: - description: |- - monitors is a collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - items: - type: string - type: array - pool: - description: |- - pool is the rados pool name. - Default is rbd. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: boolean - secretRef: - description: |- - secretRef is name of the authentication secret for RBDUser. If provided - overrides keyring. - Default is nil. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - user: - description: |- - user is the rados user name. - Default is admin. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - required: - - image - - monitors - type: object - scaleIO: - description: scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". - Default is "xfs". - type: string - gateway: - description: gateway is the host address of the ScaleIO API Gateway. - type: string - protectionDomain: - description: protectionDomain is the name of the ScaleIO Protection Domain for the configured storage. - type: string - readOnly: - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef references to the secret for ScaleIO user and other - sensitive information. If this is not provided, Login operation will fail. - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - sslEnabled: - description: sslEnabled Flag enable/disable SSL communication with Gateway, default false - type: boolean - storageMode: - description: |- - storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage Pool associated with the protection domain. - type: string - system: - description: system is the name of the storage system as configured in ScaleIO. - type: string - volumeName: - description: |- - volumeName is the name of a volume already created in the ScaleIO system - that is associated with this volume source. - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - description: |- - secret represents a secret that should populate this volume. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - properties: - defaultMode: - description: |- - defaultMode is Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values - for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items If unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - optional: - description: optional field specify whether the Secret or its keys must be defined - type: boolean - secretName: - description: |- - secretName is the name of the secret in the pod's namespace to use. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - type: string - type: object - storageos: - description: storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef specifies the secret to use for obtaining the StorageOS API - credentials. If not specified, default values will be attempted. - properties: - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? - type: string - type: object - x-kubernetes-map-type: atomic - volumeName: - description: |- - volumeName is the human-readable name of the StorageOS volume. Volume - names are only unique within a namespace. - type: string - volumeNamespace: - description: |- - volumeNamespace specifies the scope of the volume within StorageOS. If no - namespace is specified then the Pod's namespace will be used. This allows the - Kubernetes name scoping to be mirrored within StorageOS for tighter integration. - Set VolumeName to any name to override the default behaviour. - Set to "default" if you are not using namespaces within StorageOS. - Namespaces that do not pre-exist within StorageOS will be created. - type: string - type: object - vsphereVolume: - description: vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine - properties: - fsType: - description: |- - fsType is filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName. - type: string - storagePolicyName: - description: storagePolicyName is the storage Policy Based Management (SPBM) profile name. - type: string - volumePath: - description: volumePath is the path that identifies vSphere volume vmdk - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - required: - - containers - type: object - type: object - ttlSecondsAfterFinished: - description: |- - ttlSecondsAfterFinished limits the lifetime of a Job that has finished - execution (either Complete or Failed). If this field is set, - ttlSecondsAfterFinished after the Job finishes, it is eligible to be - automatically deleted. When the Job is being deleted, its lifecycle - guarantees (e.g. finalizers) will be honored. If this field is unset, - the Job won't be automatically deleted. If this field is set to zero, - the Job becomes eligible to be deleted immediately after it finishes. - format: int32 - type: integer - required: - - template - type: object - type: object - required: - - name - - template - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - startupPolicy: - description: StartupPolicy, if set, configures in what order jobs must be started - properties: - startupPolicyOrder: - description: |- - StartupPolicyOrder determines the startup order of the ReplicatedJobs. - AnyOrder means to start replicated jobs in any order. - InOrder means to start them as they are listed in the JobSet. A ReplicatedJob is started only - when all the jobs of the previous one are ready. - enum: - - AnyOrder - - InOrder - type: string - required: - - startupPolicyOrder - type: object - x-kubernetes-validations: - - message: Value is immutable - rule: self == oldSelf - successPolicy: - description: |- - SuccessPolicy configures when to declare the JobSet as - succeeded. - The JobSet is always declared succeeded if all jobs in the set - finished with status complete. - properties: - operator: - description: Operator determines either All or Any of the selected jobs should succeed to consider the JobSet successful - enum: - - All - - Any - type: string - targetReplicatedJobs: - description: |- - TargetReplicatedJobs are the names of the replicated jobs the operator will apply to. - A null or empty list will apply to all replicatedJobs. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - operator - type: object - x-kubernetes-validations: - - message: Value is immutable - rule: self == oldSelf - suspend: - description: Suspend suspends all running child Jobs when set to true. - type: boolean - ttlSecondsAfterFinished: - description: |- - TTLSecondsAfterFinished limits the lifetime of a JobSet that has finished - execution (either Complete or Failed). If this field is set, - TTLSecondsAfterFinished after the JobSet finishes, it is eligible to be - automatically deleted. When the JobSet is being deleted, its lifecycle - guarantees (e.g. finalizers) will be honored. If this field is unset, - the JobSet won't be automatically deleted. If this field is set to zero, - the JobSet becomes eligible to be deleted immediately after it finishes. - format: int32 - minimum: 0 - type: integer - type: object - status: - description: JobSetStatus defines the observed state of JobSet - properties: - conditions: - items: - description: "Condition contains details for one aspect of the current state of this API Resource.\n---\nThis struct is intended for direct use as an array at the field path .status.conditions. For example,\n\n\n\ttype FooStatus struct{\n\t // Represents the observations of a foo's current state.\n\t // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t // other fields\n\t}" - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: |- - type of condition in CamelCase or in foo.example.com/CamelCase. - --- - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be - useful (see .node.status.conditions), the ability to deconflict is important. - The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - replicatedJobsStatus: - description: ReplicatedJobsStatus track the number of JobsReady for each replicatedJob. - items: - description: ReplicatedJobStatus defines the observed ReplicatedJobs Readiness. - properties: - active: - description: |- - Active is the number of child Jobs with at least 1 pod in a running or pending state - which are not marked for deletion. - format: int32 - type: integer - failed: - description: Failed is the number of failed child Jobs. - format: int32 - type: integer - name: - description: Name of the ReplicatedJob. - type: string - ready: - description: |- - Ready is the number of child Jobs where the number of ready pods and completed pods - is greater than or equal to the total expected pod count for the Job (i.e., the minimum - of job.spec.parallelism and job.spec.completions). - format: int32 - type: integer - succeeded: - description: Succeeded is the number of successfully completed child Jobs. - format: int32 - type: integer - suspended: - description: Suspended is the number of child Jobs which are in a suspended state. - format: int32 - type: integer - required: - - active - - failed - - name - - ready - - succeeded - - suspended - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - restarts: - description: Restarts tracks the number of times the JobSet has restarted (i.e. recreated in case of RecreateAll policy). - format: int32 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: controller-manager - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: serviceaccount - app.kubernetes.io/part-of: jobset - name: jobset-controller-manager - namespace: jobset-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: leader-election-role - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: role - app.kubernetes.io/part-of: jobset - name: jobset-leader-election-role - namespace: jobset-system -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: jobset-manager-role -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - update - - watch -- apiGroups: - - admissionregistration.k8s.io - resources: - - mutatingwebhookconfigurations - verbs: - - get - - list - - update - - watch -- apiGroups: - - admissionregistration.k8s.io - resources: - - validatingwebhookconfigurations - verbs: - - get - - list - - update - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs/status - verbs: - - get - - patch - - update -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/finalizers - verbs: - - update -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/status - verbs: - - get - - patch - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: kube-rbac-proxy - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: metrics-reader - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: clusterrole - app.kubernetes.io/part-of: jobset - name: jobset-metrics-reader -rules: -- nonResourceURLs: - - /metrics - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: kube-rbac-proxy - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: proxy-role - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: clusterrole - app.kubernetes.io/part-of: jobset - name: jobset-proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: leader-election-rolebinding - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: rolebinding - app.kubernetes.io/part-of: jobset - name: jobset-leader-election-rolebinding - namespace: jobset-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: jobset-leader-election-role -subjects: -- kind: ServiceAccount - name: jobset-controller-manager - namespace: jobset-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: manager-rolebinding - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: clusterrolebinding - app.kubernetes.io/part-of: jobset - name: jobset-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: jobset-manager-role -subjects: -- kind: ServiceAccount - name: jobset-controller-manager - namespace: jobset-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: kube-rbac-proxy - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: proxy-rolebinding - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: clusterrolebinding - app.kubernetes.io/part-of: jobset - name: jobset-proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: jobset-proxy-role -subjects: -- kind: ServiceAccount - name: jobset-controller-manager - namespace: jobset-system ---- -apiVersion: v1 -kind: Secret -metadata: - name: jobset-webhook-server-cert - namespace: jobset-system ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: kube-rbac-proxy - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: controller-manager-metrics-service - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: service - app.kubernetes.io/part-of: jobset - control-plane: controller-manager - name: jobset-controller-manager-metrics-service - namespace: jobset-system -spec: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https - selector: - control-plane: controller-manager ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: webhook - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: webhook-service - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: service - app.kubernetes.io/part-of: jobset - name: jobset-webhook-service - namespace: jobset-system -spec: - ports: - - port: 443 - protocol: TCP - targetPort: 9443 - selector: - control-plane: controller-manager ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app.kubernetes.io/component: manager - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: controller-manager - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: deployment - app.kubernetes.io/part-of: jobset - control-plane: controller-manager - name: jobset-controller-manager - namespace: jobset-system -spec: - replicas: 1 - selector: - matchLabels: - control-plane: controller-manager - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: manager - labels: - control-plane: controller-manager - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - arm64 - - ppc64le - - s390x - - key: kubernetes.io/os - operator: In - values: - - linux - containers: - - args: - - --zap-log-level=2 - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 - - --leader-elect - command: - - /manager - image: registry.k8s.io/jobset/jobset:v0.5.2 - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - ports: - - containerPort: 9443 - name: webhook-server - protocol: TCP - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - cpu: 2 - memory: 512Mi - requests: - cpu: 500m - memory: 128Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - securityContext: - runAsNonRoot: true - serviceAccountName: jobset-controller-manager - terminationGracePeriodSeconds: 10 - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: jobset-webhook-server-cert - tolerations: - - effect: NoSchedule - key: components.gke.io/gke-managed-components - operator: Equal - value: "true" ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: jobset-mutating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: jobset-webhook-service - namespace: jobset-system - path: /mutate-jobset-x-k8s-io-v1alpha2-jobset - failurePolicy: Fail - name: mjobset.kb.io - rules: - - apiGroups: - - jobset.x-k8s.io - apiVersions: - - v1alpha2 - operations: - - CREATE - - UPDATE - resources: - - jobsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: jobset-webhook-service - namespace: jobset-system - path: /mutate--v1-pod - failurePolicy: Fail - name: mpod.kb.io - objectSelector: - matchExpressions: - - key: jobset.sigs.k8s.io/jobset-name - operator: Exists - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods - sideEffects: None ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: jobset-validating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: jobset-webhook-service - namespace: jobset-system - path: /validate-jobset-x-k8s-io-v1alpha2-jobset - failurePolicy: Fail - name: vjobset.kb.io - rules: - - apiGroups: - - jobset.x-k8s.io - apiVersions: - - v1alpha2 - operations: - - CREATE - - UPDATE - resources: - - jobsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: jobset-webhook-service - namespace: jobset-system - path: /validate--v1-pod - failurePolicy: Fail - name: vpod.kb.io - objectSelector: - matchExpressions: - - key: jobset.sigs.k8s.io/jobset-name - operator: Exists - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods - sideEffects: None diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/jobset-v0.7.2.yaml b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/jobset-v0.7.2.yaml deleted file mode 100644 index a600eaf308..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/jobset-v0.7.2.yaml +++ /dev/null @@ -1,8918 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Namespace -metadata: - labels: - app.kubernetes.io/component: manager - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: system - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: namespace - app.kubernetes.io/part-of: jobset - control-plane: controller-manager - name: jobset-system ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - controller-gen.kubebuilder.io/version: v0.16.2 - name: jobsets.jobset.x-k8s.io -spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - name: jobset-webhook-service - namespace: jobset-system - path: /convert - conversionReviewVersions: - - v1 - group: jobset.x-k8s.io - names: - kind: JobSet - listKind: JobSetList - plural: jobsets - singular: jobset - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Final state of JobSet - jsonPath: .status.terminalState - name: TerminalState - type: string - - description: Number of restarts - jsonPath: .status.restarts - name: Restarts - type: string - - jsonPath: .status.conditions[?(@.type=="Completed")].status - name: Completed - type: string - - description: JobSet suspended - jsonPath: .spec.suspend - name: Suspended - type: string - - description: Time this JobSet was created - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1alpha2 - schema: - openAPIV3Schema: - description: JobSet is the Schema for the jobsets API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: JobSetSpec defines the desired state of JobSet - properties: - coordinator: - description: |- - Coordinator can be used to assign a specific pod as the coordinator for - the JobSet. If defined, an annotation will be added to all Jobs and pods with - coordinator pod, which contains the stable network endpoint where the - coordinator pod can be reached. - jobset.sigs.k8s.io/coordinator=. - properties: - jobIndex: - description: |- - JobIndex is the index of Job which contains the coordinator pod - (i.e., for a ReplicatedJob with N replicas, there are Job indexes 0 to N-1). - type: integer - podIndex: - description: PodIndex is the Job completion index of the coordinator pod. - type: integer - replicatedJob: - description: |- - ReplicatedJob is the name of the ReplicatedJob which contains - the coordinator pod. - type: string - required: - - replicatedJob - type: object - failurePolicy: - description: |- - FailurePolicy, if set, configures when to declare the JobSet as - failed. - The JobSet is always declared failed if any job in the set - finished with status failed. - properties: - maxRestarts: - description: |- - MaxRestarts defines the limit on the number of JobSet restarts. - A restart is achieved by recreating all active child jobs. - format: int32 - type: integer - restartStrategy: - default: Recreate - description: |- - RestartStrategy defines the strategy to use when restarting the JobSet. - Defaults to Recreate. - enum: - - Recreate - - BlockingRecreate - type: string - rules: - description: |- - List of failure policy rules for this JobSet. - For a given Job failure, the rules will be evaluated in order, - and only the first matching rule will be executed. - If no matching rule is found, the RestartJobSet action is applied. - items: - description: |- - FailurePolicyRule defines a FailurePolicyAction to be executed if a child job - fails due to a reason listed in OnJobFailureReasons. - properties: - action: - description: The action to take if the rule is matched. - enum: - - FailJobSet - - RestartJobSet - - RestartJobSetAndIgnoreMaxRestarts - type: string - name: - description: |- - The name of the failure policy rule. - The name is defaulted to 'failurePolicyRuleN' where N is the index of the failure policy rule. - The name must match the regular expression "^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$". - type: string - onJobFailureReasons: - description: |- - The requirement on the job failure reasons. The requirement - is satisfied if at least one reason matches the list. - The rules are evaluated in order, and the first matching - rule is executed. - An empty list applies the rule to any job failure reason. - items: - type: string - type: array - targetReplicatedJobs: - description: |- - TargetReplicatedJobs are the names of the replicated jobs the operator applies to. - An empty list will apply to all replicatedJobs. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - action - - name - type: object - type: array - type: object - x-kubernetes-validations: - - message: Value is immutable - rule: self == oldSelf - managedBy: - description: |- - ManagedBy is used to indicate the controller or entity that manages a JobSet. - The built-in JobSet controller reconciles JobSets which don't have this - field at all or the field value is the reserved string - `jobset.sigs.k8s.io/jobset-controller`, but skips reconciling JobSets - with a custom value for this field. - - The value must be a valid domain-prefixed path (e.g. acme.io/foo) - - all characters before the first "/" must be a valid subdomain as defined - by RFC 1123. All characters trailing the first "/" must be valid HTTP Path - characters as defined by RFC 3986. The value cannot exceed 63 characters. - The field is immutable. - type: string - network: - description: Network defines the networking options for the jobset. - properties: - enableDNSHostnames: - description: |- - EnableDNSHostnames allows pods to be reached via their hostnames. - Pods will be reachable using the fully qualified pod hostname: - ---. - type: boolean - publishNotReadyAddresses: - description: |- - Indicates if DNS records of pods should be published before the pods are ready. - Defaults to True. - type: boolean - subdomain: - description: |- - Subdomain is an explicit choice for a network subdomain name - When set, any replicated job in the set is added to this network. - Defaults to if not set. - type: string - type: object - x-kubernetes-validations: - - message: Value is immutable - rule: self == oldSelf - replicatedJobs: - description: ReplicatedJobs is the group of jobs that will form the set. - items: - properties: - name: - description: |- - Name is the name of the entry and will be used as a suffix - for the Job name. - type: string - replicas: - default: 1 - description: |- - Replicas is the number of jobs that will be created from this ReplicatedJob's template. - Jobs names will be in the format: -- - format: int32 - type: integer - template: - description: Template defines the template of the Job that will be created. - properties: - metadata: - description: |- - Standard object's metadata of the jobs created from this template. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - Specification of the desired behavior of the job. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - activeDeadlineSeconds: - description: |- - Specifies the duration in seconds relative to the startTime that the job - may be continuously active before the system tries to terminate it; value - must be positive integer. If a Job is suspended (at creation or through an - update), this timer will effectively be stopped and reset when the Job is - resumed again. - format: int64 - type: integer - backoffLimit: - description: |- - Specifies the number of retries before marking this job failed. - Defaults to 6 - format: int32 - type: integer - backoffLimitPerIndex: - description: |- - Specifies the limit for the number of retries within an - index before marking this index as failed. When enabled the number of - failures per index is kept in the pod's - batch.kubernetes.io/job-index-failure-count annotation. It can only - be set when Job's completionMode=Indexed, and the Pod's restart - policy is Never. The field is immutable. - This field is beta-level. It can be used when the `JobBackoffLimitPerIndex` - feature gate is enabled (enabled by default). - format: int32 - type: integer - completionMode: - description: |- - completionMode specifies how Pod completions are tracked. It can be - `NonIndexed` (default) or `Indexed`. - - `NonIndexed` means that the Job is considered complete when there have - been .spec.completions successfully completed Pods. Each Pod completion is - homologous to each other. - - `Indexed` means that the Pods of a - Job get an associated completion index from 0 to (.spec.completions - 1), - available in the annotation batch.kubernetes.io/job-completion-index. - The Job is considered complete when there is one successfully completed Pod - for each index. - When value is `Indexed`, .spec.completions must be specified and - `.spec.parallelism` must be less than or equal to 10^5. - In addition, The Pod name takes the form - `$(job-name)-$(index)-$(random-string)`, - the Pod hostname takes the form `$(job-name)-$(index)`. - - More completion modes can be added in the future. - If the Job controller observes a mode that it doesn't recognize, which - is possible during upgrades due to version skew, the controller - skips updates for the Job. - type: string - completions: - description: |- - Specifies the desired number of successfully finished pods the - job should be run with. Setting to null means that the success of any - pod signals the success of all pods, and allows parallelism to have any positive - value. Setting to 1 means that parallelism is limited to 1 and the success of that - pod signals the success of the job. - More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ - format: int32 - type: integer - managedBy: - description: |- - ManagedBy field indicates the controller that manages a Job. The k8s Job - controller reconciles jobs which don't have this field at all or the field - value is the reserved string `kubernetes.io/job-controller`, but skips - reconciling Jobs with a custom value for this field. - The value must be a valid domain-prefixed path (e.g. acme.io/foo) - - all characters before the first "/" must be a valid subdomain as defined - by RFC 1123. All characters trailing the first "/" must be valid HTTP Path - characters as defined by RFC 3986. The value cannot exceed 63 characters. - This field is immutable. - - This field is alpha-level. The job controller accepts setting the field - when the feature gate JobManagedBy is enabled (disabled by default). - type: string - manualSelector: - description: |- - manualSelector controls generation of pod labels and pod selectors. - Leave `manualSelector` unset unless you are certain what you are doing. - When false or unset, the system pick labels unique to this job - and appends those labels to the pod template. When true, - the user is responsible for picking unique labels and specifying - the selector. Failure to pick a unique label may cause this - and other jobs to not function correctly. However, You may see - `manualSelector=true` in jobs that were created with the old `extensions/v1beta1` - API. - More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#specifying-your-own-pod-selector - type: boolean - maxFailedIndexes: - description: |- - Specifies the maximal number of failed indexes before marking the Job as - failed, when backoffLimitPerIndex is set. Once the number of failed - indexes exceeds this number the entire Job is marked as Failed and its - execution is terminated. When left as null the job continues execution of - all of its indexes and is marked with the `Complete` Job condition. - It can only be specified when backoffLimitPerIndex is set. - It can be null or up to completions. It is required and must be - less than or equal to 10^4 when is completions greater than 10^5. - This field is beta-level. It can be used when the `JobBackoffLimitPerIndex` - feature gate is enabled (enabled by default). - format: int32 - type: integer - parallelism: - description: |- - Specifies the maximum desired number of pods the job should - run at any given time. The actual number of pods running in steady state will - be less than this number when ((.spec.completions - .status.successful) < .spec.parallelism), - i.e. when the work left to do is less than max parallelism. - More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ - format: int32 - type: integer - podFailurePolicy: - description: |- - Specifies the policy of handling failed pods. In particular, it allows to - specify the set of actions and conditions which need to be - satisfied to take the associated action. - If empty, the default behaviour applies - the counter of failed pods, - represented by the jobs's .status.failed field, is incremented and it is - checked against the backoffLimit. This field cannot be used in combination - with restartPolicy=OnFailure. - properties: - rules: - description: |- - A list of pod failure policy rules. The rules are evaluated in order. - Once a rule matches a Pod failure, the remaining of the rules are ignored. - When no rule matches the Pod failure, the default handling applies - the - counter of pod failures is incremented and it is checked against - the backoffLimit. At most 20 elements are allowed. - items: - description: |- - PodFailurePolicyRule describes how a pod failure is handled when the requirements are met. - One of onExitCodes and onPodConditions, but not both, can be used in each rule. - properties: - action: - description: |- - Specifies the action taken on a pod failure when the requirements are satisfied. - Possible values are: - - - FailJob: indicates that the pod's job is marked as Failed and all - running pods are terminated. - - FailIndex: indicates that the pod's index is marked as Failed and will - not be restarted. - This value is beta-level. It can be used when the - `JobBackoffLimitPerIndex` feature gate is enabled (enabled by default). - - Ignore: indicates that the counter towards the .backoffLimit is not - incremented and a replacement pod is created. - - Count: indicates that the pod is handled in the default way - the - counter towards the .backoffLimit is incremented. - Additional values are considered to be added in the future. Clients should - react to an unknown action by skipping the rule. - type: string - onExitCodes: - description: Represents the requirement on the container exit codes. - properties: - containerName: - description: |- - Restricts the check for exit codes to the container with the - specified name. When null, the rule applies to all containers. - When specified, it should match one the container or initContainer - names in the pod template. - type: string - operator: - description: |- - Represents the relationship between the container exit code(s) and the - specified values. Containers completed with success (exit code 0) are - excluded from the requirement check. Possible values are: - - - In: the requirement is satisfied if at least one container exit code - (might be multiple if there are multiple containers not restricted - by the 'containerName' field) is in the set of specified values. - - NotIn: the requirement is satisfied if at least one container exit code - (might be multiple if there are multiple containers not restricted - by the 'containerName' field) is not in the set of specified values. - Additional values are considered to be added in the future. Clients should - react to an unknown operator by assuming the requirement is not satisfied. - type: string - values: - description: |- - Specifies the set of values. Each returned container exit code (might be - multiple in case of multiple containers) is checked against this set of - values with respect to the operator. The list of values must be ordered - and must not contain duplicates. Value '0' cannot be used for the In operator. - At least one element is required. At most 255 elements are allowed. - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - - values - type: object - onPodConditions: - description: |- - Represents the requirement on the pod conditions. The requirement is represented - as a list of pod condition patterns. The requirement is satisfied if at - least one pattern matches an actual pod condition. At most 20 elements are allowed. - items: - description: |- - PodFailurePolicyOnPodConditionsPattern describes a pattern for matching - an actual pod condition type. - properties: - status: - description: |- - Specifies the required Pod condition status. To match a pod condition - it is required that the specified status equals the pod condition status. - Defaults to True. - type: string - type: - description: |- - Specifies the required Pod condition type. To match a pod condition - it is required that specified type equals the pod condition type. - type: string - required: - - status - - type - type: object - type: array - x-kubernetes-list-type: atomic - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic - required: - - rules - type: object - podReplacementPolicy: - description: |- - podReplacementPolicy specifies when to create replacement Pods. - Possible values are: - - TerminatingOrFailed means that we recreate pods - when they are terminating (has a metadata.deletionTimestamp) or failed. - - Failed means to wait until a previously created Pod is fully terminated (has phase - Failed or Succeeded) before creating a replacement Pod. - - When using podFailurePolicy, Failed is the the only allowed value. - TerminatingOrFailed and Failed are allowed values when podFailurePolicy is not in use. - This is an beta field. To use this, enable the JobPodReplacementPolicy feature toggle. - This is on by default. - type: string - selector: - description: |- - A label query over pods that should match the pod count. - Normally, the system sets this field for you. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - successPolicy: - description: |- - successPolicy specifies the policy when the Job can be declared as succeeded. - If empty, the default behavior applies - the Job is declared as succeeded - only when the number of succeeded pods equals to the completions. - When the field is specified, it must be immutable and works only for the Indexed Jobs. - Once the Job meets the SuccessPolicy, the lingering pods are terminated. - - This field is beta-level. To use this field, you must enable the - `JobSuccessPolicy` feature gate (enabled by default). - properties: - rules: - description: |- - rules represents the list of alternative rules for the declaring the Jobs - as successful before `.status.succeeded >= .spec.completions`. Once any of the rules are met, - the "SucceededCriteriaMet" condition is added, and the lingering pods are removed. - The terminal state for such a Job has the "Complete" condition. - Additionally, these rules are evaluated in order; Once the Job meets one of the rules, - other rules are ignored. At most 20 elements are allowed. - items: - description: |- - SuccessPolicyRule describes rule for declaring a Job as succeeded. - Each rule must have at least one of the "succeededIndexes" or "succeededCount" specified. - properties: - succeededCount: - description: |- - succeededCount specifies the minimal required size of the actual set of the succeeded indexes - for the Job. When succeededCount is used along with succeededIndexes, the check is - constrained only to the set of indexes specified by succeededIndexes. - For example, given that succeededIndexes is "1-4", succeededCount is "3", - and completed indexes are "1", "3", and "5", the Job isn't declared as succeeded - because only "1" and "3" indexes are considered in that rules. - When this field is null, this doesn't default to any value and - is never evaluated at any time. - When specified it needs to be a positive integer. - format: int32 - type: integer - succeededIndexes: - description: |- - succeededIndexes specifies the set of indexes - which need to be contained in the actual set of the succeeded indexes for the Job. - The list of indexes must be within 0 to ".spec.completions-1" and - must not contain duplicates. At least one element is required. - The indexes are represented as intervals separated by commas. - The intervals can be a decimal integer or a pair of decimal integers separated by a hyphen. - The number are listed in represented by the first and last element of the series, - separated by a hyphen. - For example, if the completed indexes are 1, 3, 4, 5 and 7, they are - represented as "1,3-5,7". - When this field is null, this field doesn't default to any value - and is never evaluated at any time. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - required: - - rules - type: object - suspend: - description: |- - suspend specifies whether the Job controller should create Pods or not. If - a Job is created with suspend set to true, no Pods are created by the Job - controller. If a Job is suspended after creation (i.e. the flag goes from - false to true), the Job controller will delete all active Pods associated - with this Job. Users must design their workload to gracefully handle this. - Suspending a Job will reset the StartTime field of the Job, effectively - resetting the ActiveDeadlineSeconds timer too. Defaults to false. - type: boolean - template: - description: |- - Describes the pod that will be created when executing a job. - The only allowed template.spec.restartPolicy values are "Never" or "OnFailure". - More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ - properties: - metadata: - description: |- - Standard object's metadata. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - Specification of the desired behavior of the pod. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - activeDeadlineSeconds: - description: |- - Optional duration in seconds the pod may be active on the node relative to - StartTime before the system will actively try to mark it failed and kill associated containers. - Value must be a positive integer. - format: int64 - type: integer - affinity: - description: If specified, the pod's scheduling constraints - properties: - nodeAffinity: - description: Describes node affinity scheduling rules for the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node matches the corresponding matchExpressions; the - node(s) with the highest sum are the most preferred. - items: - description: |- - An empty preferred scheduling term matches all objects with implicit weight 0 - (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated with the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector requirements by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. - items: - description: |- - A null or empty node selector term matches no objects. The requirements of - them are ANDed. - The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector requirements by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the anti-affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the anti-affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the anti-affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - automountServiceAccountToken: - description: AutomountServiceAccountToken indicates whether a service account token should be automatically mounted. - type: boolean - containers: - description: |- - List of containers belonging to the pod. - Containers cannot currently be added or removed. - There must be at least one container in a Pod. - Cannot be updated. - items: - description: A single application container that you want to run within a pod. - properties: - args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment variable present in a Container. - properties: - name: - description: Name of the environment variable. Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's namespace - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration that the container should sleep before being terminated. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration that the container should sleep before being terminated. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. - type: string - ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. - type: string - securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to the container. - type: string - role: - description: Role is a SELinux role label that applies to the container. - type: string - type: - description: Type is a SELinux type label that applies to the container. - type: string - user: - description: User is a SELinux user label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. - items: - description: volumeDevice describes a mapping of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. - type: string - name: - description: name must match the name of a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. - items: - description: VolumeMount describes a mounting of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - dnsConfig: - description: |- - Specifies the DNS parameters of a pod. - Parameters specified here will be merged to the generated DNS - configuration based on DNSPolicy. - properties: - nameservers: - description: |- - A list of DNS name server IP addresses. - This will be appended to the base nameservers generated from DNSPolicy. - Duplicated nameservers will be removed. - items: - type: string - type: array - x-kubernetes-list-type: atomic - options: - description: |- - A list of DNS resolver options. - This will be merged with the base options generated from DNSPolicy. - Duplicated entries will be removed. Resolution options given in Options - will override those that appear in the base DNSPolicy. - items: - description: PodDNSConfigOption defines DNS resolver options of a pod. - properties: - name: - description: Required. - type: string - value: - type: string - type: object - type: array - x-kubernetes-list-type: atomic - searches: - description: |- - A list of DNS search domains for host-name lookup. - This will be appended to the base search paths generated from DNSPolicy. - Duplicated search paths will be removed. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - dnsPolicy: - description: |- - Set DNS policy for the pod. - Defaults to "ClusterFirst". - Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. - DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. - To have DNS options set along with hostNetwork, you have to specify DNS policy - explicitly to 'ClusterFirstWithHostNet'. - type: string - enableServiceLinks: - description: |- - EnableServiceLinks indicates whether information about services should be injected into pod's - environment variables, matching the syntax of Docker links. - Optional: Defaults to true. - type: boolean - ephemeralContainers: - description: |- - List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing - pod to perform user-initiated actions such as debugging. This list cannot be specified when - creating a pod, and it cannot be modified by updating the pod spec. In order to add an - ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. - items: - description: |- - An EphemeralContainer is a temporary container that you may add to an existing Pod for - user-initiated activities such as debugging. Ephemeral containers have no resource or - scheduling guarantees, and they will not be restarted when they exit or when a Pod is - removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the - Pod to exceed its resource allocation. - - To add an ephemeral container, use the ephemeralcontainers subresource of an existing - Pod. Ephemeral containers may not be removed or restarted. - properties: - args: - description: |- - Arguments to the entrypoint. - The image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment variable present in a Container. - properties: - name: - description: Name of the environment variable. Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's namespace - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: Lifecycle is not allowed for ephemeral containers. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration that the container should sleep before being terminated. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration that the container should sleep before being terminated. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: Probes are not allowed for ephemeral containers. - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the ephemeral container specified as a DNS_LABEL. - This name must be unique among all containers, init containers and ephemeral containers. - type: string - ports: - description: Ports are not allowed for ephemeral containers. - items: - description: ContainerPort represents a network port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: Probes are not allowed for ephemeral containers. - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources - already allocated to the pod. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - Restart policy for the container to manage the restart behavior of each - container within a pod. - This may only be set for init containers. You cannot set this field on - ephemeral containers. - type: string - securityContext: - description: |- - Optional: SecurityContext defines the security options the ephemeral container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to the container. - type: string - role: - description: Role is a SELinux role label that applies to the container. - type: string - type: - description: Type is a SELinux type label that applies to the container. - type: string - user: - description: User is a SELinux user label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: Probes are not allowed for ephemeral containers. - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - targetContainerName: - description: |- - If set, the name of the container from PodSpec that this ephemeral container targets. - The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. - If not set then the ephemeral container uses the namespaces configured in the Pod spec. - - The container runtime must implement support for this feature. If the runtime does not - support namespace targeting then the result of setting this field is undefined. - type: string - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. - items: - description: volumeDevice describes a mapping of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. - type: string - name: - description: name must match the name of a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. - Cannot be updated. - items: - description: VolumeMount describes a mounting of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - hostAliases: - description: |- - HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts - file if specified. - items: - description: |- - HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the - pod's hosts file. - properties: - hostnames: - description: Hostnames for the above IP address. - items: - type: string - type: array - x-kubernetes-list-type: atomic - ip: - description: IP address of the host file entry. - type: string - required: - - ip - type: object - type: array - x-kubernetes-list-map-keys: - - ip - x-kubernetes-list-type: map - hostIPC: - description: |- - Use the host's ipc namespace. - Optional: Default to false. - type: boolean - hostNetwork: - description: |- - Host networking requested for this pod. Use the host's network namespace. - If this option is set, the ports that will be used must be specified. - Default to false. - type: boolean - hostPID: - description: |- - Use the host's pid namespace. - Optional: Default to false. - type: boolean - hostUsers: - description: |- - Use the host's user namespace. - Optional: Default to true. - If set to true or not present, the pod will be run in the host user namespace, useful - for when the pod needs a feature only available to the host user namespace, such as - loading a kernel module with CAP_SYS_MODULE. - When set to false, a new userns is created for the pod. Setting false is useful for - mitigating container breakout vulnerabilities even allowing users to run their - containers as root without actually having root privileges on the host. - This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. - type: boolean - hostname: - description: |- - Specifies the hostname of the Pod - If not specified, the pod's hostname will be set to a system-defined value. - type: string - imagePullSecrets: - description: |- - ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. - If specified, these secrets will be passed to individual puller implementations for them to use. - More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod - items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - initContainers: - description: |- - List of initialization containers belonging to the pod. - Init containers are executed in order prior to containers being started. If any - init container fails, the pod is considered to have failed and is handled according - to its restartPolicy. The name for an init container or normal container must be - unique among all containers. - Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. - The resourceRequirements of an init container are taken into account during scheduling - by finding the highest request/limit for each resource type, and then using the max of - of that value or the sum of the normal containers. Limits are applied to init containers - in a similar fashion. - Init containers cannot currently be added or removed. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ - items: - description: A single application container that you want to run within a pod. - properties: - args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment variable present in a Container. - properties: - name: - description: Name of the environment variable. Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's namespace - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration that the container should sleep before being terminated. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration that the container should sleep before being terminated. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. - type: string - ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. - type: string - securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to the container. - type: string - role: - description: Role is a SELinux role label that applies to the container. - type: string - type: - description: Type is a SELinux type label that applies to the container. - type: string - user: - description: User is a SELinux user label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving a GRPC port. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. - items: - description: volumeDevice describes a mapping of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. - type: string - name: - description: name must match the name of a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. - items: - description: VolumeMount describes a mounting of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - nodeName: - description: |- - NodeName indicates in which node this pod is scheduled. - If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. - Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. - This field should not be used to express a desire for the pod to be scheduled on a specific node. - https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename - type: string - nodeSelector: - additionalProperties: - type: string - description: |- - NodeSelector is a selector which must be true for the pod to fit on a node. - Selector which must match a node's labels for the pod to be scheduled on that node. - More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - type: object - x-kubernetes-map-type: atomic - os: - description: |- - Specifies the OS of the containers in the pod. - Some pod and container fields are restricted if this is set. - - If the OS field is set to linux, the following fields must be unset: - -securityContext.windowsOptions - - If the OS field is set to windows, following fields must be unset: - - spec.hostPID - - spec.hostIPC - - spec.hostUsers - - spec.securityContext.appArmorProfile - - spec.securityContext.seLinuxOptions - - spec.securityContext.seccompProfile - - spec.securityContext.fsGroup - - spec.securityContext.fsGroupChangePolicy - - spec.securityContext.sysctls - - spec.shareProcessNamespace - - spec.securityContext.runAsUser - - spec.securityContext.runAsGroup - - spec.securityContext.supplementalGroups - - spec.securityContext.supplementalGroupsPolicy - - spec.containers[*].securityContext.appArmorProfile - - spec.containers[*].securityContext.seLinuxOptions - - spec.containers[*].securityContext.seccompProfile - - spec.containers[*].securityContext.capabilities - - spec.containers[*].securityContext.readOnlyRootFilesystem - - spec.containers[*].securityContext.privileged - - spec.containers[*].securityContext.allowPrivilegeEscalation - - spec.containers[*].securityContext.procMount - - spec.containers[*].securityContext.runAsUser - - spec.containers[*].securityContext.runAsGroup - properties: - name: - description: |- - Name is the name of the operating system. The currently supported values are linux and windows. - Additional value may be defined in future and can be one of: - https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration - Clients should expect to handle additional values and treat unrecognized values in this field as os: null - type: string - required: - - name - type: object - overhead: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. - This field will be autopopulated at admission time by the RuntimeClass admission controller. If - the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. - The RuntimeClass admission controller will reject Pod create requests which have the overhead already - set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value - defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. - More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md - type: object - preemptionPolicy: - description: |- - PreemptionPolicy is the Policy for preempting pods with lower priority. - One of Never, PreemptLowerPriority. - Defaults to PreemptLowerPriority if unset. - type: string - priority: - description: |- - The priority value. Various system components use this field to find the - priority of the pod. When Priority Admission Controller is enabled, it - prevents users from setting this field. The admission controller populates - this field from PriorityClassName. - The higher the value, the higher the priority. - format: int32 - type: integer - priorityClassName: - description: |- - If specified, indicates the pod's priority. "system-node-critical" and - "system-cluster-critical" are two special keywords which indicate the - highest priorities with the former being the highest priority. Any other - name must be defined by creating a PriorityClass object with that name. - If not specified, the pod priority will be default or zero if there is no - default. - type: string - readinessGates: - description: |- - If specified, all readiness gates will be evaluated for pod readiness. - A pod is ready when all its containers are ready AND - all conditions specified in the readiness gates have status equal to "True" - More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates - items: - description: PodReadinessGate contains the reference to a pod condition - properties: - conditionType: - description: ConditionType refers to a condition in the pod's condition list with matching type. - type: string - required: - - conditionType - type: object - type: array - x-kubernetes-list-type: atomic - resourceClaims: - description: |- - ResourceClaims defines which ResourceClaims must be allocated - and reserved before the Pod is allowed to start. The resources - will be made available to those containers which consume them - by name. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. - items: - description: |- - PodResourceClaim references exactly one ResourceClaim, either directly - or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim - for the pod. - - It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. - Containers that need access to the ResourceClaim reference it with this name. - properties: - name: - description: |- - Name uniquely identifies this resource claim inside the pod. - This must be a DNS_LABEL. - type: string - resourceClaimName: - description: |- - ResourceClaimName is the name of a ResourceClaim object in the same - namespace as this pod. - - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. - type: string - resourceClaimTemplateName: - description: |- - ResourceClaimTemplateName is the name of a ResourceClaimTemplate - object in the same namespace as this pod. - - The template will be used to create a new ResourceClaim, which will - be bound to this pod. When this pod is deleted, the ResourceClaim - will also be deleted. The pod name and resource name, along with a - generated component, will be used to form a unique name for the - ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. - - This field is immutable and no changes will be made to the - corresponding ResourceClaim by the control plane after creating the - ResourceClaim. - - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - restartPolicy: - description: |- - Restart policy for all containers within the pod. - One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. - Default to Always. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy - type: string - runtimeClassName: - description: |- - RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used - to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. - If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an - empty definition that uses the default runtime handler. - More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class - type: string - schedulerName: - description: |- - If specified, the pod will be dispatched by specified scheduler. - If not specified, the pod will be dispatched by default scheduler. - type: string - schedulingGates: - description: |- - SchedulingGates is an opaque list of values that if specified will block scheduling the pod. - If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the - scheduler will not attempt to schedule the pod. - - SchedulingGates can only be set at pod creation time, and be removed only afterwards. - items: - description: PodSchedulingGate is associated to a Pod to guard its scheduling. - properties: - name: - description: |- - Name of the scheduling gate. - Each scheduling gate must have a unique name field. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - securityContext: - description: |- - SecurityContext holds pod-level security attributes and common container settings. - Optional: Defaults to empty. See type description for default values of each field. - properties: - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - fsGroup: - description: |- - A special supplemental group that applies to all containers in a pod. - Some volume types allow the Kubelet to change the ownership of that volume - to be owned by the pod: - - 1. The owning GID will be the FSGroup - 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) - 3. The permission bits are OR'd with rw-rw---- - - If unset, the Kubelet will not modify the ownership and permissions of any volume. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - fsGroupChangePolicy: - description: |- - fsGroupChangePolicy defines behavior of changing ownership and permission of the volume - before being exposed inside Pod. This field will only apply to - volume types which support fsGroup based ownership(and permissions). - It will have no effect on ephemeral volume types such as: secret, configmaps - and emptydir. - Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. - Note that this field cannot be set when spec.os.name is windows. - type: string - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in SecurityContext. If set in - both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to the container. - type: string - role: - description: Role is a SELinux role label that applies to the container. - type: string - type: - description: Type is a SELinux type label that applies to the container. - type: string - user: - description: User is a SELinux user label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - supplementalGroups: - description: |- - A list of groups applied to the first process run in each container, in - addition to the container's primary GID and fsGroup (if specified). If - the SupplementalGroupsPolicy feature is enabled, the - supplementalGroupsPolicy field determines whether these are in addition - to or instead of any group memberships defined in the container image. - If unspecified, no additional groups are added, though group memberships - defined in the container image may still be used, depending on the - supplementalGroupsPolicy field. - Note that this field cannot be set when spec.os.name is windows. - items: - format: int64 - type: integer - type: array - x-kubernetes-list-type: atomic - supplementalGroupsPolicy: - description: |- - Defines how supplemental groups of the first container processes are calculated. - Valid values are "Merge" and "Strict". If not specified, "Merge" is used. - (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled - and the container runtime must implement support for this feature. - Note that this field cannot be set when spec.os.name is windows. - type: string - sysctls: - description: |- - Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported - sysctls (by the container runtime) might fail to launch. - Note that this field cannot be set when spec.os.name is windows. - items: - description: Sysctl defines a kernel parameter to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options within a container's SecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - serviceAccount: - description: |- - DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. - Deprecated: Use serviceAccountName instead. - type: string - serviceAccountName: - description: |- - ServiceAccountName is the name of the ServiceAccount to use to run this pod. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ - type: string - setHostnameAsFQDN: - description: |- - If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). - In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). - In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. - If a pod does not have FQDN, this has no effect. - Default to false. - type: boolean - shareProcessNamespace: - description: |- - Share a single process namespace between all of the containers in a pod. - When this is set containers will be able to view and signal processes from other containers - in the same pod, and the first process in each container will not be assigned PID 1. - HostPID and ShareProcessNamespace cannot both be set. - Optional: Default to false. - type: boolean - subdomain: - description: |- - If specified, the fully qualified Pod hostname will be "...svc.". - If not specified, the pod will not have a domainname at all. - type: string - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - If this value is nil, the default grace period will be used instead. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - Defaults to 30 seconds. - format: int64 - type: integer - tolerations: - description: If specified, the pod's tolerations. - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - topologySpreadConstraints: - description: |- - TopologySpreadConstraints describes how a group of pods ought to spread across topology - domains. Scheduler will schedule pods in a way which abides by the constraints. - All topologySpreadConstraints are ANDed. - items: - description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. - properties: - labelSelector: - description: |- - LabelSelector is used to find matching pods. - Pods that match this label selector are counted to determine the number of pods - in their corresponding topology domain. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select the pods over which - spreading will be calculated. The keys are used to lookup values from the - incoming pod labels, those key-value labels are ANDed with labelSelector - to select the group of existing pods over which spreading will be calculated - for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - MatchLabelKeys cannot be set when LabelSelector isn't set. - Keys that don't exist in the incoming pod labels will - be ignored. A null or empty list means only match against labelSelector. - - This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxSkew: - description: |- - MaxSkew describes the degree to which pods may be unevenly distributed. - When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference - between the number of matching pods in the target topology and the global minimum. - The global minimum is the minimum number of matching pods in an eligible domain - or zero if the number of eligible domains is less than MinDomains. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 2/2/1: - In this case, the global minimum is 1. - | zone1 | zone2 | zone3 | - | P P | P P | P | - - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; - scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) - violate MaxSkew(1). - - if MaxSkew is 2, incoming pod can be scheduled onto any zone. - When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence - to topologies that satisfy it. - It's a required field. Default value is 1 and 0 is not allowed. - format: int32 - type: integer - minDomains: - description: |- - MinDomains indicates a minimum number of eligible domains. - When the number of eligible domains with matching topology keys is less than minDomains, - Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. - And when the number of eligible domains with matching topology keys equals or greater than minDomains, - this value has no effect on scheduling. - As a result, when the number of eligible domains is less than minDomains, - scheduler won't schedule more than maxSkew Pods to those domains. - If value is nil, the constraint behaves as if MinDomains is equal to 1. - Valid values are integers greater than 0. - When value is not nil, WhenUnsatisfiable must be DoNotSchedule. - - For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same - labelSelector spread as 2/2/2: - | zone1 | zone2 | zone3 | - | P P | P P | P P | - The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. - In this situation, new pod with the same labelSelector cannot be scheduled, - because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, - it will violate MaxSkew. - format: int32 - type: integer - nodeAffinityPolicy: - description: |- - NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector - when calculating pod topology spread skew. Options are: - - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. - - If this value is nil, the behavior is equivalent to the Honor policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. - type: string - nodeTaintsPolicy: - description: |- - NodeTaintsPolicy indicates how we will treat node taints when calculating - pod topology spread skew. Options are: - - Honor: nodes without taints, along with tainted nodes for which the incoming pod - has a toleration, are included. - - Ignore: node taints are ignored. All nodes are included. - - If this value is nil, the behavior is equivalent to the Ignore policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. - type: string - topologyKey: - description: |- - TopologyKey is the key of node labels. Nodes that have a label with this key - and identical values are considered to be in the same topology. - We consider each as a "bucket", and try to put balanced number - of pods into each bucket. - We define a domain as a particular instance of a topology. - Also, we define an eligible domain as a domain whose nodes meet the requirements of - nodeAffinityPolicy and nodeTaintsPolicy. - e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. - And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. - It's a required field. - type: string - whenUnsatisfiable: - description: |- - WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy - the spread constraint. - - DoNotSchedule (default) tells the scheduler not to schedule it. - - ScheduleAnyway tells the scheduler to schedule the pod in any location, - but giving higher precedence to topologies that would help reduce the - skew. - A constraint is considered "Unsatisfiable" for an incoming pod - if and only if every possible node assignment for that pod would violate - "MaxSkew" on some topology. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 3/1/1: - | zone1 | zone2 | zone3 | - | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled - to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies - MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler - won't make it *more* imbalanced. - It's a required field. - type: string - required: - - maxSkew - - topologyKey - - whenUnsatisfiable - type: object - type: array - x-kubernetes-list-map-keys: - - topologyKey - - whenUnsatisfiable - x-kubernetes-list-type: map - volumes: - description: |- - List of volumes that can be mounted by containers belonging to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes - items: - description: Volume represents a named volume in a pod that may be accessed by any container in the pod. - properties: - awsElasticBlockStore: - description: |- - awsElasticBlockStore represents an AWS Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - format: int32 - type: integer - readOnly: - description: |- - readOnly value true will force the readOnly setting in VolumeMounts. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: boolean - volumeID: - description: |- - volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: string - required: - - volumeID - type: object - azureDisk: - description: azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. - properties: - cachingMode: - description: 'cachingMode is the Host Caching mode: None, Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data disk in the blob storage - type: string - diskURI: - description: diskURI is the URI of data disk in the blob storage - type: string - fsType: - default: ext4 - description: |- - fsType is Filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - kind: - description: 'kind expected values are Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared' - type: string - readOnly: - default: false - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: azureFile represents an Azure File Service mount on the host and bind mount to the pod. - properties: - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret that contains Azure Storage Account Name and Key - type: string - shareName: - description: shareName is the azure share Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: cephFS represents a Ceph FS mount on the host that shares a pod's lifetime - properties: - monitors: - description: |- - monitors is Required: Monitors is a collection of Ceph monitors - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - items: - type: string - type: array - x-kubernetes-list-type: atomic - path: - description: 'path is Optional: Used as the mounted root, rather than the full Ceph tree, default is /' - type: string - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: boolean - secretFile: - description: |- - secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - secretRef: - description: |- - secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - user: - description: |- - user is optional: User is the rados user name, default is admin - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - required: - - monitors - type: object - cinder: - description: |- - cinder represents a cinder volume attached and mounted on kubelets host machine. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: boolean - secretRef: - description: |- - secretRef is optional: points to a secret object containing parameters used to connect - to OpenStack. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - volumeID: - description: |- - volumeID used to identify the volume in cinder. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap that should populate this volume - properties: - defaultMode: - description: |- - defaultMode is optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional specify whether the ConfigMap or its keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - csi: - description: csi (Container Storage Interface) represents ephemeral storage that is handled by certain external CSI drivers (Beta feature). - properties: - driver: - description: |- - driver is the name of the CSI driver that handles this volume. - Consult with your admin for the correct name as registered in the cluster. - type: string - fsType: - description: |- - fsType to mount. Ex. "ext4", "xfs", "ntfs". - If not provided, the empty value is passed to the associated CSI driver - which will determine the default filesystem to apply. - type: string - nodePublishSecretRef: - description: |- - nodePublishSecretRef is a reference to the secret object containing - sensitive information to pass to the CSI driver to complete the CSI - NodePublishVolume and NodeUnpublishVolume calls. - This field is optional, and may be empty if no secret is required. If the - secret object contains more than one secret, all secret references are passed. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - readOnly: - description: |- - readOnly specifies a read-only configuration for the volume. - Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: |- - volumeAttributes stores driver-specific properties that are passed to the CSI - driver. Consult your driver's documentation for supported values. - type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API about the pod that should populate this volume - properties: - defaultMode: - description: |- - Optional: mode bits to use on created files by default. Must be a - Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: Items is a list of downward API volume file - items: - description: DownwardAPIVolumeFile represents information to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - emptyDir: - description: |- - emptyDir represents a temporary directory that shares a pod's lifetime. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - properties: - medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: |- - ephemeral represents a volume that is handled by a cluster storage driver. - The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, - and deleted when the pod is removed. - - Use this if: - a) the volume is only needed while the pod runs, - b) features of normal volumes like restoring from snapshot or capacity - tracking are needed, - c) the storage driver is specified through a storage class, and - d) the storage driver supports dynamic volume provisioning through - a PersistentVolumeClaim (see EphemeralVolumeSource for more - information on the connection between this volume type - and PersistentVolumeClaim). - - Use PersistentVolumeClaim or one of the vendor-specific - APIs for volumes that persist for longer than the lifecycle - of an individual pod. - - Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to - be used that way - see the documentation of the driver for - more information. - - A pod can use both types of ephemeral volumes and - persistent volumes at the same time. - properties: - volumeClaimTemplate: - description: |- - Will be used to create a stand-alone PVC to provision the volume. - The pod in which this EphemeralVolumeSource is embedded will be the - owner of the PVC, i.e. the PVC will be deleted together with the - pod. The name of the PVC will be `-` where - `` is the name from the `PodSpec.Volumes` array - entry. Pod validation will reject the pod if the concatenated name - is not valid for a PVC (for example, too long). - - An existing PVC with that name that is not owned by the pod - will *not* be used for the pod to avoid using an unrelated - volume by mistake. Starting the pod is then blocked until - the unrelated PVC is removed. If such a pre-created PVC is - meant to be used by the pod, the PVC has to updated with an - owner reference to the pod once the pod exists. Normally - this should not be necessary, but it may be useful when - manually reconstructing a broken cluster. - - This field is read-only and no changes will be made by Kubernetes - to the PVC after it has been created. - - Required, must not be nil. - properties: - metadata: - description: |- - May contain labels and annotations that will be copied into the PVC - when creating it. No other fields are allowed and will be rejected during - validation. - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - The specification for the PersistentVolumeClaim. The entire content is - copied unchanged into the PVC that gets created from this - template. The same fields as in a PersistentVolumeClaim - are also valid here. - properties: - accessModes: - description: |- - accessModes contains the desired access modes the volume should have. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 - items: - type: string - type: array - x-kubernetes-list-type: atomic - dataSource: - description: |- - dataSource field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - If the provisioner or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified data source. - When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, - and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. - If the namespace is specified, then dataSourceRef will not be copied to dataSource. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: |- - dataSourceRef specifies the object from which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a non-empty API group (non - core object) or a PersistentVolumeClaim object. - When this field is specified, volume binding will only succeed if the type of - the specified object matches some installed volume populator or dynamic - provisioner. - This field will replace the functionality of the dataSource field and as such - if both fields are non-empty, they must have the same value. For backwards - compatibility, when namespace isn't specified in dataSourceRef, - both fields (dataSource and dataSourceRef) will be set to the same - value automatically if one of them is empty and the other is non-empty. - When namespace is specified in dataSourceRef, - dataSource isn't set to the same value and must be empty. - There are three important differences between dataSource and dataSourceRef: - * While dataSource only allows two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping them), dataSourceRef - preserves all values, and generates an error if a disallowed value is - specified. - * While dataSource only allows local objects, dataSourceRef allows objects - in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. - (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - namespace: - description: |- - Namespace is the namespace of resource being referenced - Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. - (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: |- - resources represents the minimum resources the volume should have. - If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements - that are lower than previous value but must still be higher than capacity recorded in the - status field of the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - description: selector is a label query over volumes to consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: |- - storageClassName is the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 - type: string - volumeAttributesClassName: - description: |- - volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. - If specified, the CSI driver will create or update the volume with the attributes defined - in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. - If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be - set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource - exists. - More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). - type: string - volumeMode: - description: |- - volumeMode defines what type of volume is required by the claim. - Value of Filesystem is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference to the PersistentVolume backing this claim. - type: string - type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - lun: - description: 'lun is Optional: FC target lun number' - format: int32 - type: integer - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target worldwide names (WWNs)' - items: - type: string - type: array - x-kubernetes-list-type: atomic - wwids: - description: |- - wwids Optional: FC volume world wide identifiers (wwids) - Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - flexVolume: - description: |- - flexVolume represents a generic volume resource that is - provisioned/attached using an exec based plugin. - properties: - driver: - description: driver is the name of the driver to use for this volume. - type: string - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field holds extra command options if any.' - type: object - readOnly: - description: |- - readOnly is Optional: defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef is Optional: secretRef is reference to the secret object containing - sensitive information to pass to the plugin scripts. This may be - empty if no secret object is specified. If the secret object - contains more than one secret, all secrets are passed to the plugin - scripts. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - required: - - driver - type: object - flocker: - description: flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running - properties: - datasetName: - description: |- - datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker - should be considered as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the dataset. This is unique identifier of a Flocker dataset - type: string - type: object - gcePersistentDisk: - description: |- - gcePersistentDisk represents a GCE Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - properties: - fsType: - description: |- - fsType is filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - format: int32 - type: integer - pdName: - description: |- - pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: boolean - required: - - pdName - type: object - gitRepo: - description: |- - gitRepo represents a git repository at a particular revision. - DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an - EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir - into the Pod's container. - properties: - directory: - description: |- - directory is the target directory name. - Must not contain or start with '..'. If '.' is supplied, the volume directory will be the - git repository. Otherwise, if specified, the volume will contain the git repository in - the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for the specified revision. - type: string - required: - - repository - type: object - glusterfs: - description: |- - glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. - More info: https://examples.k8s.io/volumes/glusterfs/README.md - properties: - endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: string - path: - description: |- - path is the Glusterfs volume path. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: string - readOnly: - description: |- - readOnly here will force the Glusterfs volume to be mounted with read-only permissions. - Defaults to false. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: boolean - required: - - endpoints - - path - type: object - hostPath: - description: |- - hostPath represents a pre-existing file or directory on the host - machine that is directly exposed to the container. This is generally - used for system agents or other privileged things that are allowed - to see the host machine. Most containers will NOT need this. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - properties: - path: - description: |- - path of the directory on the host. - If the path is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - type: - description: |- - type for HostPath Volume - Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - required: - - path - type: object - image: - description: |- - image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. - The volume is resolved at pod startup depending on which PullPolicy value is provided: - - - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - - The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. - A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. - The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. - The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. - The volume will be mounted read-only (ro) and non-executable files (noexec). - Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). - The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. - properties: - pullPolicy: - description: |- - Policy for pulling OCI objects. Possible values are: - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - type: string - reference: - description: |- - Required: Image or artifact reference to be used. - Behaves in the same way as pod.spec.containers[*].image. - Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - type: object - iscsi: - description: |- - iscsi represents an ISCSI Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md - properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether support iSCSI Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether support iSCSI Session CHAP authentication - type: boolean - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - type: string - initiatorName: - description: |- - initiatorName is the custom iSCSI Initiator Name. - If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface - : will be created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified Name. - type: string - iscsiInterface: - default: default - description: |- - iscsiInterface is the interface Name that uses an iSCSI transport. - Defaults to 'default' (tcp). - type: string - lun: - description: lun represents iSCSI Target Lun number. - format: int32 - type: integer - portals: - description: |- - portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - items: - type: string - type: array - x-kubernetes-list-type: atomic - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret for iSCSI target and initiator authentication - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - targetPortal: - description: |- - targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - description: |- - name of the volume. - Must be a DNS_LABEL and unique within the pod. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - nfs: - description: |- - nfs represents an NFS mount on the host that shares a pod's lifetime - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - properties: - path: - description: |- - path that is exported by the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - readOnly: - description: |- - readOnly here will force the NFS export to be mounted with read-only permissions. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: boolean - server: - description: |- - server is the hostname or IP address of the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: |- - persistentVolumeClaimVolumeSource represents a reference to a - PersistentVolumeClaim in the same namespace. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - properties: - claimName: - description: |- - claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - type: string - readOnly: - description: |- - readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - pdID: - description: pdID is the ID that identifies Photon Controller persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: portworxVolume represents a portworx volume attached and mounted on kubelets host machine - properties: - fsType: - description: |- - fSType represents the filesystem type to mount - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources secrets, configmaps, and downward API - properties: - defaultMode: - description: |- - defaultMode are the mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - sources: - description: |- - sources is the list of volume projections. Each entry in this list - handles one source. - items: - description: |- - Projection that may be projected along with other supported volume types. - Exactly one of these fields must be set. - properties: - clusterTrustBundle: - description: |- - ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field - of ClusterTrustBundle objects in an auto-updating file. - - Alpha, gated by the ClusterTrustBundleProjection feature gate. - - ClusterTrustBundle objects can either be selected by name, or by the - combination of signer name and a label selector. - - Kubelet performs aggressive normalization of the PEM contents written - into the pod filesystem. Esoteric PEM features such as inter-block - comments and block headers are stripped. Certificates are deduplicated. - The ordering of certificates within the file is arbitrary, and Kubelet - may change the order over time. - properties: - labelSelector: - description: |- - Select all ClusterTrustBundles that match this label selector. Only has - effect if signerName is set. Mutually-exclusive with name. If unset, - interpreted as "match nothing". If set but empty, interpreted as "match - everything". - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - name: - description: |- - Select a single ClusterTrustBundle by object name. Mutually-exclusive - with signerName and labelSelector. - type: string - optional: - description: |- - If true, don't block pod startup if the referenced ClusterTrustBundle(s) - aren't available. If using name, then the named ClusterTrustBundle is - allowed not to exist. If using signerName, then the combination of - signerName and labelSelector is allowed to match zero - ClusterTrustBundles. - type: boolean - path: - description: Relative path from the volume root to write the bundle. - type: string - signerName: - description: |- - Select all ClusterTrustBundles that match this signer name. - Mutually-exclusive with name. The contents of all selected - ClusterTrustBundles will be unified and deduplicated. - type: string - required: - - path - type: object - configMap: - description: configMap information about the configMap data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional specify whether the ConfigMap or its keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - downwardAPI: - description: downwardAPI information about the downwardAPI data to project - properties: - items: - description: Items is a list of DownwardAPIVolume file - items: - description: DownwardAPIVolumeFile represents information to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - secret: - description: secret information about the secret data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional field specify whether the Secret or its key must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - serviceAccountToken: - description: serviceAccountToken is information about the serviceAccountToken data to project - properties: - audience: - description: |- - audience is the intended audience of the token. A recipient of a token - must identify itself with an identifier specified in the audience of the - token, and otherwise should reject the token. The audience defaults to the - identifier of the apiserver. - type: string - expirationSeconds: - description: |- - expirationSeconds is the requested duration of validity of the service - account token. As the token approaches expiration, the kubelet volume - plugin will proactively rotate the service account token. The kubelet will - start trying to rotate the token if the token is older than 80 percent of - its time to live or if the token is older than 24 hours.Defaults to 1 hour - and must be at least 10 minutes. - format: int64 - type: integer - path: - description: |- - path is the path relative to the mount point of the file to project the - token into. - type: string - required: - - path - type: object - type: object - type: array - x-kubernetes-list-type: atomic - type: object - quobyte: - description: quobyte represents a Quobyte mount on the host that shares a pod's lifetime - properties: - group: - description: |- - group to map volume access to - Default is no group - type: string - readOnly: - description: |- - readOnly here will force the Quobyte volume to be mounted with read-only permissions. - Defaults to false. - type: boolean - registry: - description: |- - registry represents a single or multiple Quobyte Registry services - specified as a string as host:port pair (multiple entries are separated with commas) - which acts as the central registry for volumes - type: string - tenant: - description: |- - tenant owning the given Quobyte volume in the Backend - Used with dynamically provisioned Quobyte volumes, value is set by the plugin - type: string - user: - description: |- - user to map volume access to - Defaults to serivceaccount user - type: string - volume: - description: volume is a string that references an already created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: |- - rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. - More info: https://examples.k8s.io/volumes/rbd/README.md - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - type: string - image: - description: |- - image is the rados image name. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - keyring: - default: /etc/ceph/keyring - description: |- - keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - monitors: - description: |- - monitors is a collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - items: - type: string - type: array - x-kubernetes-list-type: atomic - pool: - default: rbd - description: |- - pool is the rados pool name. - Default is rbd. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: boolean - secretRef: - description: |- - secretRef is name of the authentication secret for RBDUser. If provided - overrides keyring. - Default is nil. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - user: - default: admin - description: |- - user is the rados user name. - Default is admin. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - required: - - image - - monitors - type: object - scaleIO: - description: scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. - properties: - fsType: - default: xfs - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". - Default is "xfs". - type: string - gateway: - description: gateway is the host address of the ScaleIO API Gateway. - type: string - protectionDomain: - description: protectionDomain is the name of the ScaleIO Protection Domain for the configured storage. - type: string - readOnly: - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef references to the secret for ScaleIO user and other - sensitive information. If this is not provided, Login operation will fail. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - sslEnabled: - description: sslEnabled Flag enable/disable SSL communication with Gateway, default false - type: boolean - storageMode: - default: ThinProvisioned - description: |- - storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage Pool associated with the protection domain. - type: string - system: - description: system is the name of the storage system as configured in ScaleIO. - type: string - volumeName: - description: |- - volumeName is the name of a volume already created in the ScaleIO system - that is associated with this volume source. - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - description: |- - secret represents a secret that should populate this volume. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - properties: - defaultMode: - description: |- - defaultMode is Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values - for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items If unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - optional: - description: optional field specify whether the Secret or its keys must be defined - type: boolean - secretName: - description: |- - secretName is the name of the secret in the pod's namespace to use. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - type: string - type: object - storageos: - description: storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef specifies the secret to use for obtaining the StorageOS API - credentials. If not specified, default values will be attempted. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - volumeName: - description: |- - volumeName is the human-readable name of the StorageOS volume. Volume - names are only unique within a namespace. - type: string - volumeNamespace: - description: |- - volumeNamespace specifies the scope of the volume within StorageOS. If no - namespace is specified then the Pod's namespace will be used. This allows the - Kubernetes name scoping to be mirrored within StorageOS for tighter integration. - Set VolumeName to any name to override the default behaviour. - Set to "default" if you are not using namespaces within StorageOS. - Namespaces that do not pre-exist within StorageOS will be created. - type: string - type: object - vsphereVolume: - description: vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine - properties: - fsType: - description: |- - fsType is filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName. - type: string - storagePolicyName: - description: storagePolicyName is the storage Policy Based Management (SPBM) profile name. - type: string - volumePath: - description: volumePath is the path that identifies vSphere volume vmdk - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - containers - type: object - type: object - ttlSecondsAfterFinished: - description: |- - ttlSecondsAfterFinished limits the lifetime of a Job that has finished - execution (either Complete or Failed). If this field is set, - ttlSecondsAfterFinished after the Job finishes, it is eligible to be - automatically deleted. When the Job is being deleted, its lifecycle - guarantees (e.g. finalizers) will be honored. If this field is unset, - the Job won't be automatically deleted. If this field is set to zero, - the Job becomes eligible to be deleted immediately after it finishes. - format: int32 - type: integer - required: - - template - type: object - type: object - required: - - name - - template - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - startupPolicy: - description: StartupPolicy, if set, configures in what order jobs must be started - properties: - startupPolicyOrder: - description: |- - StartupPolicyOrder determines the startup order of the ReplicatedJobs. - AnyOrder means to start replicated jobs in any order. - InOrder means to start them as they are listed in the JobSet. A ReplicatedJob is started only - when all the jobs of the previous one are ready. - enum: - - AnyOrder - - InOrder - type: string - required: - - startupPolicyOrder - type: object - x-kubernetes-validations: - - message: Value is immutable - rule: self == oldSelf - successPolicy: - description: |- - SuccessPolicy configures when to declare the JobSet as - succeeded. - The JobSet is always declared succeeded if all jobs in the set - finished with status complete. - properties: - operator: - description: Operator determines either All or Any of the selected jobs should succeed to consider the JobSet successful - enum: - - All - - Any - type: string - targetReplicatedJobs: - description: |- - TargetReplicatedJobs are the names of the replicated jobs the operator will apply to. - A null or empty list will apply to all replicatedJobs. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - operator - type: object - x-kubernetes-validations: - - message: Value is immutable - rule: self == oldSelf - suspend: - description: Suspend suspends all running child Jobs when set to true. - type: boolean - ttlSecondsAfterFinished: - description: |- - TTLSecondsAfterFinished limits the lifetime of a JobSet that has finished - execution (either Complete or Failed). If this field is set, - TTLSecondsAfterFinished after the JobSet finishes, it is eligible to be - automatically deleted. When the JobSet is being deleted, its lifecycle - guarantees (e.g. finalizers) will be honored. If this field is unset, - the JobSet won't be automatically deleted. If this field is set to zero, - the JobSet becomes eligible to be deleted immediately after it finishes. - format: int32 - minimum: 0 - type: integer - type: object - status: - description: JobSetStatus defines the observed state of JobSet - properties: - conditions: - items: - description: Condition contains details for one aspect of the current state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - replicatedJobsStatus: - description: ReplicatedJobsStatus track the number of JobsReady for each replicatedJob. - items: - description: ReplicatedJobStatus defines the observed ReplicatedJobs Readiness. - properties: - active: - description: |- - Active is the number of child Jobs with at least 1 pod in a running or pending state - which are not marked for deletion. - format: int32 - type: integer - failed: - description: Failed is the number of failed child Jobs. - format: int32 - type: integer - name: - description: Name of the ReplicatedJob. - type: string - ready: - description: |- - Ready is the number of child Jobs where the number of ready pods and completed pods - is greater than or equal to the total expected pod count for the Job (i.e., the minimum - of job.spec.parallelism and job.spec.completions). - format: int32 - type: integer - succeeded: - description: Succeeded is the number of successfully completed child Jobs. - format: int32 - type: integer - suspended: - description: Suspended is the number of child Jobs which are in a suspended state. - format: int32 - type: integer - required: - - active - - failed - - name - - ready - - succeeded - - suspended - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - restarts: - description: Restarts tracks the number of times the JobSet has restarted (i.e. recreated in case of RecreateAll policy). - format: int32 - type: integer - restartsCountTowardsMax: - description: RestartsCountTowardsMax tracks the number of times the JobSet has restarted that counts towards the maximum allowed number of restarts. - format: int32 - type: integer - terminalState: - description: |- - TerminalState the state of the JobSet when it finishes execution. - It can be either Complete or Failed. Otherwise, it is empty by default. - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: controller-manager - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: serviceaccount - app.kubernetes.io/part-of: jobset - name: jobset-controller-manager - namespace: jobset-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: leader-election-role - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: role - app.kubernetes.io/part-of: jobset - name: jobset-leader-election-role - namespace: jobset-system -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: jobset-manager-role -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - pods - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - update - - watch -- apiGroups: - - admissionregistration.k8s.io - resources: - - mutatingwebhookconfigurations - - validatingwebhookconfigurations - verbs: - - get - - list - - update - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs/status - verbs: - - get - - patch - - update -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/finalizers - verbs: - - update -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/status - verbs: - - get - - patch - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: kube-rbac-proxy - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: metrics-reader - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: clusterrole - app.kubernetes.io/part-of: jobset - name: jobset-metrics-reader -rules: -- nonResourceURLs: - - /metrics - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: kube-rbac-proxy - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: proxy-role - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: clusterrole - app.kubernetes.io/part-of: jobset - name: jobset-proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: leader-election-rolebinding - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: rolebinding - app.kubernetes.io/part-of: jobset - name: jobset-leader-election-rolebinding - namespace: jobset-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: jobset-leader-election-role -subjects: -- kind: ServiceAccount - name: jobset-controller-manager - namespace: jobset-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: manager-rolebinding - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: clusterrolebinding - app.kubernetes.io/part-of: jobset - name: jobset-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: jobset-manager-role -subjects: -- kind: ServiceAccount - name: jobset-controller-manager - namespace: jobset-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: kube-rbac-proxy - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: proxy-rolebinding - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: clusterrolebinding - app.kubernetes.io/part-of: jobset - name: jobset-proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: jobset-proxy-role -subjects: -- kind: ServiceAccount - name: jobset-controller-manager - namespace: jobset-system ---- -apiVersion: v1 -kind: Secret -metadata: - name: jobset-webhook-server-cert - namespace: jobset-system ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: kube-rbac-proxy - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: controller-manager-metrics-service - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: service - app.kubernetes.io/part-of: jobset - control-plane: controller-manager - name: jobset-controller-manager-metrics-service - namespace: jobset-system -spec: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https - selector: - control-plane: controller-manager ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: webhook - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: webhook-service - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: service - app.kubernetes.io/part-of: jobset - name: jobset-webhook-service - namespace: jobset-system -spec: - ports: - - port: 443 - protocol: TCP - targetPort: 9443 - selector: - control-plane: controller-manager ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app.kubernetes.io/component: manager - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: controller-manager - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: deployment - app.kubernetes.io/part-of: jobset - control-plane: controller-manager - name: jobset-controller-manager - namespace: jobset-system -spec: - replicas: 1 - selector: - matchLabels: - control-plane: controller-manager - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: manager - labels: - control-plane: controller-manager - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - arm64 - - ppc64le - - s390x - - key: kubernetes.io/os - operator: In - values: - - linux - containers: - - args: - - --zap-log-level=2 - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 - - --leader-elect - command: - - /manager - image: registry.k8s.io/jobset/jobset:v0.7.2 - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - ports: - - containerPort: 9443 - name: webhook-server - protocol: TCP - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - cpu: 2 - memory: 512Mi - requests: - cpu: 500m - memory: 128Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - securityContext: - runAsNonRoot: true - serviceAccountName: jobset-controller-manager - terminationGracePeriodSeconds: 10 - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: jobset-webhook-server-cert - tolerations: - - effect: NoSchedule - key: components.gke.io/gke-managed-components - operator: Equal - value: "true" ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: jobset-mutating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: jobset-webhook-service - namespace: jobset-system - path: /mutate-jobset-x-k8s-io-v1alpha2-jobset - failurePolicy: Fail - name: mjobset.kb.io - rules: - - apiGroups: - - jobset.x-k8s.io - apiVersions: - - v1alpha2 - operations: - - CREATE - - UPDATE - resources: - - jobsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: jobset-webhook-service - namespace: jobset-system - path: /mutate--v1-pod - failurePolicy: Fail - name: mpod.kb.io - objectSelector: - matchExpressions: - - key: jobset.sigs.k8s.io/jobset-name - operator: Exists - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods - sideEffects: None ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: jobset-validating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: jobset-webhook-service - namespace: jobset-system - path: /validate-jobset-x-k8s-io-v1alpha2-jobset - failurePolicy: Fail - name: vjobset.kb.io - rules: - - apiGroups: - - jobset.x-k8s.io - apiVersions: - - v1alpha2 - operations: - - CREATE - - UPDATE - resources: - - jobsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: jobset-webhook-service - namespace: jobset-system - path: /validate--v1-pod - failurePolicy: Fail - name: vpod.kb.io - objectSelector: - matchExpressions: - - key: jobset.sigs.k8s.io/jobset-name - operator: Exists - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods - sideEffects: None diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/jobset-v0.8.1.yaml b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/jobset-v0.8.1.yaml deleted file mode 100644 index f5b0d82245..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/jobset-v0.8.1.yaml +++ /dev/null @@ -1,9075 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Namespace -metadata: - labels: - app.kubernetes.io/component: manager - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: system - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: namespace - app.kubernetes.io/part-of: jobset - control-plane: controller-manager - name: jobset-system ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - controller-gen.kubebuilder.io/version: v0.16.2 - name: jobsets.jobset.x-k8s.io -spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - name: jobset-webhook-service - namespace: jobset-system - path: /convert - conversionReviewVersions: - - v1 - group: jobset.x-k8s.io - names: - kind: JobSet - listKind: JobSetList - plural: jobsets - singular: jobset - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Final state of JobSet - jsonPath: .status.terminalState - name: TerminalState - type: string - - description: Number of restarts - jsonPath: .status.restarts - name: Restarts - type: string - - jsonPath: .status.conditions[?(@.type=="Completed")].status - name: Completed - type: string - - description: JobSet suspended - jsonPath: .spec.suspend - name: Suspended - type: string - - description: Time this JobSet was created - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1alpha2 - schema: - openAPIV3Schema: - description: JobSet is the Schema for the jobsets API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: JobSetSpec defines the desired state of JobSet - properties: - coordinator: - description: |- - Coordinator can be used to assign a specific pod as the coordinator for - the JobSet. If defined, an annotation will be added to all Jobs and pods with - coordinator pod, which contains the stable network endpoint where the - coordinator pod can be reached. - jobset.sigs.k8s.io/coordinator=. - properties: - jobIndex: - description: |- - JobIndex is the index of Job which contains the coordinator pod - (i.e., for a ReplicatedJob with N replicas, there are Job indexes 0 to N-1). - type: integer - podIndex: - description: PodIndex is the Job completion index of the coordinator pod. - type: integer - replicatedJob: - description: |- - ReplicatedJob is the name of the ReplicatedJob which contains - the coordinator pod. - type: string - required: - - replicatedJob - type: object - failurePolicy: - description: |- - FailurePolicy, if set, configures when to declare the JobSet as - failed. - The JobSet is always declared failed if any job in the set - finished with status failed. - properties: - maxRestarts: - description: |- - MaxRestarts defines the limit on the number of JobSet restarts. - A restart is achieved by recreating all active child jobs. - format: int32 - type: integer - restartStrategy: - default: Recreate - description: |- - RestartStrategy defines the strategy to use when restarting the JobSet. - Defaults to Recreate. - enum: - - Recreate - - BlockingRecreate - type: string - rules: - description: |- - List of failure policy rules for this JobSet. - For a given Job failure, the rules will be evaluated in order, - and only the first matching rule will be executed. - If no matching rule is found, the RestartJobSet action is applied. - items: - description: |- - FailurePolicyRule defines a FailurePolicyAction to be executed if a child job - fails due to a reason listed in OnJobFailureReasons. - properties: - action: - description: The action to take if the rule is matched. - enum: - - FailJobSet - - RestartJobSet - - RestartJobSetAndIgnoreMaxRestarts - type: string - name: - description: |- - The name of the failure policy rule. - The name is defaulted to 'failurePolicyRuleN' where N is the index of the failure policy rule. - The name must match the regular expression "^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$". - type: string - onJobFailureReasons: - description: |- - The requirement on the job failure reasons. The requirement - is satisfied if at least one reason matches the list. - The rules are evaluated in order, and the first matching - rule is executed. - An empty list applies the rule to any job failure reason. - items: - type: string - type: array - targetReplicatedJobs: - description: |- - TargetReplicatedJobs are the names of the replicated jobs the operator applies to. - An empty list will apply to all replicatedJobs. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - action - - name - type: object - type: array - type: object - x-kubernetes-validations: - - message: Value is immutable - rule: self == oldSelf - managedBy: - description: |- - ManagedBy is used to indicate the controller or entity that manages a JobSet. - The built-in JobSet controller reconciles JobSets which don't have this - field at all or the field value is the reserved string - `jobset.sigs.k8s.io/jobset-controller`, but skips reconciling JobSets - with a custom value for this field. - - The value must be a valid domain-prefixed path (e.g. acme.io/foo) - - all characters before the first "/" must be a valid subdomain as defined - by RFC 1123. All characters trailing the first "/" must be valid HTTP Path - characters as defined by RFC 3986. The value cannot exceed 63 characters. - The field is immutable. - type: string - network: - description: Network defines the networking options for the jobset. - properties: - enableDNSHostnames: - description: |- - EnableDNSHostnames allows pods to be reached via their hostnames. - Pods will be reachable using the fully qualified pod hostname: - ---. - type: boolean - publishNotReadyAddresses: - description: |- - Indicates if DNS records of pods should be published before the pods are ready. - Defaults to True. - type: boolean - subdomain: - description: |- - Subdomain is an explicit choice for a network subdomain name - When set, any replicated job in the set is added to this network. - Defaults to if not set. - type: string - type: object - x-kubernetes-validations: - - message: Value is immutable - rule: self == oldSelf - replicatedJobs: - description: ReplicatedJobs is the group of jobs that will form the set. - items: - properties: - dependsOn: - description: |- - DependsOn is an optional list that specifies the preceding ReplicatedJobs upon which - the current ReplicatedJob depends. If specified, the ReplicatedJob will be created - only after the referenced ReplicatedJobs reach their desired state. - The Order of ReplicatedJobs is defined by their enumeration in the slice. - Note, that the first ReplicatedJob in the slice cannot use the DependsOn API. - Currently, only a single item is supported in the DependsOn list. - If JobSet is suspended the all active ReplicatedJobs will be suspended. When JobSet is - resumed the Job sequence starts again. - This API is mutually exclusive with the StartupPolicy API. - items: - description: DependsOn defines the dependency on the previous ReplicatedJob status. - properties: - name: - description: Name of the previous ReplicatedJob. - type: string - status: - description: Status defines the condition for the ReplicatedJob. Only Ready or Complete status can be set. - enum: - - Ready - - Complete - type: string - required: - - name - - status - type: object - maxItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - x-kubernetes-validations: - - message: Value is immutable - rule: self == oldSelf - name: - description: |- - Name is the name of the entry and will be used as a suffix - for the Job name. - type: string - replicas: - default: 1 - description: |- - Replicas is the number of jobs that will be created from this ReplicatedJob's template. - Jobs names will be in the format: -- - format: int32 - type: integer - template: - description: Template defines the template of the Job that will be created. - properties: - metadata: - description: |- - Standard object's metadata of the jobs created from this template. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - Specification of the desired behavior of the job. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - activeDeadlineSeconds: - description: |- - Specifies the duration in seconds relative to the startTime that the job - may be continuously active before the system tries to terminate it; value - must be positive integer. If a Job is suspended (at creation or through an - update), this timer will effectively be stopped and reset when the Job is - resumed again. - format: int64 - type: integer - backoffLimit: - description: |- - Specifies the number of retries before marking this job failed. - Defaults to 6 - format: int32 - type: integer - backoffLimitPerIndex: - description: |- - Specifies the limit for the number of retries within an - index before marking this index as failed. When enabled the number of - failures per index is kept in the pod's - batch.kubernetes.io/job-index-failure-count annotation. It can only - be set when Job's completionMode=Indexed, and the Pod's restart - policy is Never. The field is immutable. - This field is beta-level. It can be used when the `JobBackoffLimitPerIndex` - feature gate is enabled (enabled by default). - format: int32 - type: integer - completionMode: - description: |- - completionMode specifies how Pod completions are tracked. It can be - `NonIndexed` (default) or `Indexed`. - - `NonIndexed` means that the Job is considered complete when there have - been .spec.completions successfully completed Pods. Each Pod completion is - homologous to each other. - - `Indexed` means that the Pods of a - Job get an associated completion index from 0 to (.spec.completions - 1), - available in the annotation batch.kubernetes.io/job-completion-index. - The Job is considered complete when there is one successfully completed Pod - for each index. - When value is `Indexed`, .spec.completions must be specified and - `.spec.parallelism` must be less than or equal to 10^5. - In addition, The Pod name takes the form - `$(job-name)-$(index)-$(random-string)`, - the Pod hostname takes the form `$(job-name)-$(index)`. - - More completion modes can be added in the future. - If the Job controller observes a mode that it doesn't recognize, which - is possible during upgrades due to version skew, the controller - skips updates for the Job. - type: string - completions: - description: |- - Specifies the desired number of successfully finished pods the - job should be run with. Setting to null means that the success of any - pod signals the success of all pods, and allows parallelism to have any positive - value. Setting to 1 means that parallelism is limited to 1 and the success of that - pod signals the success of the job. - More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ - format: int32 - type: integer - managedBy: - description: |- - ManagedBy field indicates the controller that manages a Job. The k8s Job - controller reconciles jobs which don't have this field at all or the field - value is the reserved string `kubernetes.io/job-controller`, but skips - reconciling Jobs with a custom value for this field. - The value must be a valid domain-prefixed path (e.g. acme.io/foo) - - all characters before the first "/" must be a valid subdomain as defined - by RFC 1123. All characters trailing the first "/" must be valid HTTP Path - characters as defined by RFC 3986. The value cannot exceed 63 characters. - This field is immutable. - - This field is beta-level. The job controller accepts setting the field - when the feature gate JobManagedBy is enabled (enabled by default). - type: string - manualSelector: - description: |- - manualSelector controls generation of pod labels and pod selectors. - Leave `manualSelector` unset unless you are certain what you are doing. - When false or unset, the system pick labels unique to this job - and appends those labels to the pod template. When true, - the user is responsible for picking unique labels and specifying - the selector. Failure to pick a unique label may cause this - and other jobs to not function correctly. However, You may see - `manualSelector=true` in jobs that were created with the old `extensions/v1beta1` - API. - More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#specifying-your-own-pod-selector - type: boolean - maxFailedIndexes: - description: |- - Specifies the maximal number of failed indexes before marking the Job as - failed, when backoffLimitPerIndex is set. Once the number of failed - indexes exceeds this number the entire Job is marked as Failed and its - execution is terminated. When left as null the job continues execution of - all of its indexes and is marked with the `Complete` Job condition. - It can only be specified when backoffLimitPerIndex is set. - It can be null or up to completions. It is required and must be - less than or equal to 10^4 when is completions greater than 10^5. - This field is beta-level. It can be used when the `JobBackoffLimitPerIndex` - feature gate is enabled (enabled by default). - format: int32 - type: integer - parallelism: - description: |- - Specifies the maximum desired number of pods the job should - run at any given time. The actual number of pods running in steady state will - be less than this number when ((.spec.completions - .status.successful) < .spec.parallelism), - i.e. when the work left to do is less than max parallelism. - More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ - format: int32 - type: integer - podFailurePolicy: - description: |- - Specifies the policy of handling failed pods. In particular, it allows to - specify the set of actions and conditions which need to be - satisfied to take the associated action. - If empty, the default behaviour applies - the counter of failed pods, - represented by the jobs's .status.failed field, is incremented and it is - checked against the backoffLimit. This field cannot be used in combination - with restartPolicy=OnFailure. - properties: - rules: - description: |- - A list of pod failure policy rules. The rules are evaluated in order. - Once a rule matches a Pod failure, the remaining of the rules are ignored. - When no rule matches the Pod failure, the default handling applies - the - counter of pod failures is incremented and it is checked against - the backoffLimit. At most 20 elements are allowed. - items: - description: |- - PodFailurePolicyRule describes how a pod failure is handled when the requirements are met. - One of onExitCodes and onPodConditions, but not both, can be used in each rule. - properties: - action: - description: |- - Specifies the action taken on a pod failure when the requirements are satisfied. - Possible values are: - - - FailJob: indicates that the pod's job is marked as Failed and all - running pods are terminated. - - FailIndex: indicates that the pod's index is marked as Failed and will - not be restarted. - This value is beta-level. It can be used when the - `JobBackoffLimitPerIndex` feature gate is enabled (enabled by default). - - Ignore: indicates that the counter towards the .backoffLimit is not - incremented and a replacement pod is created. - - Count: indicates that the pod is handled in the default way - the - counter towards the .backoffLimit is incremented. - Additional values are considered to be added in the future. Clients should - react to an unknown action by skipping the rule. - type: string - onExitCodes: - description: Represents the requirement on the container exit codes. - properties: - containerName: - description: |- - Restricts the check for exit codes to the container with the - specified name. When null, the rule applies to all containers. - When specified, it should match one the container or initContainer - names in the pod template. - type: string - operator: - description: |- - Represents the relationship between the container exit code(s) and the - specified values. Containers completed with success (exit code 0) are - excluded from the requirement check. Possible values are: - - - In: the requirement is satisfied if at least one container exit code - (might be multiple if there are multiple containers not restricted - by the 'containerName' field) is in the set of specified values. - - NotIn: the requirement is satisfied if at least one container exit code - (might be multiple if there are multiple containers not restricted - by the 'containerName' field) is not in the set of specified values. - Additional values are considered to be added in the future. Clients should - react to an unknown operator by assuming the requirement is not satisfied. - type: string - values: - description: |- - Specifies the set of values. Each returned container exit code (might be - multiple in case of multiple containers) is checked against this set of - values with respect to the operator. The list of values must be ordered - and must not contain duplicates. Value '0' cannot be used for the In operator. - At least one element is required. At most 255 elements are allowed. - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - - values - type: object - onPodConditions: - description: |- - Represents the requirement on the pod conditions. The requirement is represented - as a list of pod condition patterns. The requirement is satisfied if at - least one pattern matches an actual pod condition. At most 20 elements are allowed. - items: - description: |- - PodFailurePolicyOnPodConditionsPattern describes a pattern for matching - an actual pod condition type. - properties: - status: - description: |- - Specifies the required Pod condition status. To match a pod condition - it is required that the specified status equals the pod condition status. - Defaults to True. - type: string - type: - description: |- - Specifies the required Pod condition type. To match a pod condition - it is required that specified type equals the pod condition type. - type: string - required: - - status - - type - type: object - type: array - x-kubernetes-list-type: atomic - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic - required: - - rules - type: object - podReplacementPolicy: - description: |- - podReplacementPolicy specifies when to create replacement Pods. - Possible values are: - - TerminatingOrFailed means that we recreate pods - when they are terminating (has a metadata.deletionTimestamp) or failed. - - Failed means to wait until a previously created Pod is fully terminated (has phase - Failed or Succeeded) before creating a replacement Pod. - - When using podFailurePolicy, Failed is the the only allowed value. - TerminatingOrFailed and Failed are allowed values when podFailurePolicy is not in use. - This is an beta field. To use this, enable the JobPodReplacementPolicy feature toggle. - This is on by default. - type: string - selector: - description: |- - A label query over pods that should match the pod count. - Normally, the system sets this field for you. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - successPolicy: - description: |- - successPolicy specifies the policy when the Job can be declared as succeeded. - If empty, the default behavior applies - the Job is declared as succeeded - only when the number of succeeded pods equals to the completions. - When the field is specified, it must be immutable and works only for the Indexed Jobs. - Once the Job meets the SuccessPolicy, the lingering pods are terminated. - - This field is beta-level. To use this field, you must enable the - `JobSuccessPolicy` feature gate (enabled by default). - properties: - rules: - description: |- - rules represents the list of alternative rules for the declaring the Jobs - as successful before `.status.succeeded >= .spec.completions`. Once any of the rules are met, - the "SucceededCriteriaMet" condition is added, and the lingering pods are removed. - The terminal state for such a Job has the "Complete" condition. - Additionally, these rules are evaluated in order; Once the Job meets one of the rules, - other rules are ignored. At most 20 elements are allowed. - items: - description: |- - SuccessPolicyRule describes rule for declaring a Job as succeeded. - Each rule must have at least one of the "succeededIndexes" or "succeededCount" specified. - properties: - succeededCount: - description: |- - succeededCount specifies the minimal required size of the actual set of the succeeded indexes - for the Job. When succeededCount is used along with succeededIndexes, the check is - constrained only to the set of indexes specified by succeededIndexes. - For example, given that succeededIndexes is "1-4", succeededCount is "3", - and completed indexes are "1", "3", and "5", the Job isn't declared as succeeded - because only "1" and "3" indexes are considered in that rules. - When this field is null, this doesn't default to any value and - is never evaluated at any time. - When specified it needs to be a positive integer. - format: int32 - type: integer - succeededIndexes: - description: |- - succeededIndexes specifies the set of indexes - which need to be contained in the actual set of the succeeded indexes for the Job. - The list of indexes must be within 0 to ".spec.completions-1" and - must not contain duplicates. At least one element is required. - The indexes are represented as intervals separated by commas. - The intervals can be a decimal integer or a pair of decimal integers separated by a hyphen. - The number are listed in represented by the first and last element of the series, - separated by a hyphen. - For example, if the completed indexes are 1, 3, 4, 5 and 7, they are - represented as "1,3-5,7". - When this field is null, this field doesn't default to any value - and is never evaluated at any time. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - required: - - rules - type: object - suspend: - description: |- - suspend specifies whether the Job controller should create Pods or not. If - a Job is created with suspend set to true, no Pods are created by the Job - controller. If a Job is suspended after creation (i.e. the flag goes from - false to true), the Job controller will delete all active Pods associated - with this Job. Users must design their workload to gracefully handle this. - Suspending a Job will reset the StartTime field of the Job, effectively - resetting the ActiveDeadlineSeconds timer too. Defaults to false. - type: boolean - template: - description: |- - Describes the pod that will be created when executing a job. - The only allowed template.spec.restartPolicy values are "Never" or "OnFailure". - More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ - properties: - metadata: - description: |- - Standard object's metadata. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - Specification of the desired behavior of the pod. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - activeDeadlineSeconds: - description: |- - Optional duration in seconds the pod may be active on the node relative to - StartTime before the system will actively try to mark it failed and kill associated containers. - Value must be a positive integer. - format: int64 - type: integer - affinity: - description: If specified, the pod's scheduling constraints - properties: - nodeAffinity: - description: Describes node affinity scheduling rules for the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node matches the corresponding matchExpressions; the - node(s) with the highest sum are the most preferred. - items: - description: |- - An empty preferred scheduling term matches all objects with implicit weight 0 - (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated with the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector requirements by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. - items: - description: |- - A null or empty node selector term matches no objects. The requirements of - them are ANDed. - The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector requirements by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the anti-affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the anti-affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the anti-affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - automountServiceAccountToken: - description: AutomountServiceAccountToken indicates whether a service account token should be automatically mounted. - type: boolean - containers: - description: |- - List of containers belonging to the pod. - Containers cannot currently be added or removed. - There must be at least one container in a Pod. - Cannot be updated. - items: - description: A single application container that you want to run within a pod. - properties: - args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment variable present in a Container. - properties: - name: - description: Name of the environment variable. Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's namespace - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration that the container should sleep. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration that the container should sleep. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. - type: string - ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. - type: string - securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to the container. - type: string - role: - description: Role is a SELinux role label that applies to the container. - type: string - type: - description: Type is a SELinux type label that applies to the container. - type: string - user: - description: User is a SELinux user label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. - items: - description: volumeDevice describes a mapping of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. - type: string - name: - description: name must match the name of a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. - items: - description: VolumeMount describes a mounting of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - dnsConfig: - description: |- - Specifies the DNS parameters of a pod. - Parameters specified here will be merged to the generated DNS - configuration based on DNSPolicy. - properties: - nameservers: - description: |- - A list of DNS name server IP addresses. - This will be appended to the base nameservers generated from DNSPolicy. - Duplicated nameservers will be removed. - items: - type: string - type: array - x-kubernetes-list-type: atomic - options: - description: |- - A list of DNS resolver options. - This will be merged with the base options generated from DNSPolicy. - Duplicated entries will be removed. Resolution options given in Options - will override those that appear in the base DNSPolicy. - items: - description: PodDNSConfigOption defines DNS resolver options of a pod. - properties: - name: - description: |- - Name is this DNS resolver option's name. - Required. - type: string - value: - description: Value is this DNS resolver option's value. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - searches: - description: |- - A list of DNS search domains for host-name lookup. - This will be appended to the base search paths generated from DNSPolicy. - Duplicated search paths will be removed. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - dnsPolicy: - description: |- - Set DNS policy for the pod. - Defaults to "ClusterFirst". - Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. - DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. - To have DNS options set along with hostNetwork, you have to specify DNS policy - explicitly to 'ClusterFirstWithHostNet'. - type: string - enableServiceLinks: - description: |- - EnableServiceLinks indicates whether information about services should be injected into pod's - environment variables, matching the syntax of Docker links. - Optional: Defaults to true. - type: boolean - ephemeralContainers: - description: |- - List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing - pod to perform user-initiated actions such as debugging. This list cannot be specified when - creating a pod, and it cannot be modified by updating the pod spec. In order to add an - ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. - items: - description: |- - An EphemeralContainer is a temporary container that you may add to an existing Pod for - user-initiated activities such as debugging. Ephemeral containers have no resource or - scheduling guarantees, and they will not be restarted when they exit or when a Pod is - removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the - Pod to exceed its resource allocation. - - To add an ephemeral container, use the ephemeralcontainers subresource of an existing - Pod. Ephemeral containers may not be removed or restarted. - properties: - args: - description: |- - Arguments to the entrypoint. - The image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment variable present in a Container. - properties: - name: - description: Name of the environment variable. Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's namespace - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: Lifecycle is not allowed for ephemeral containers. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration that the container should sleep. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration that the container should sleep. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: Probes are not allowed for ephemeral containers. - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the ephemeral container specified as a DNS_LABEL. - This name must be unique among all containers, init containers and ephemeral containers. - type: string - ports: - description: Ports are not allowed for ephemeral containers. - items: - description: ContainerPort represents a network port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: Probes are not allowed for ephemeral containers. - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources - already allocated to the pod. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - Restart policy for the container to manage the restart behavior of each - container within a pod. - This may only be set for init containers. You cannot set this field on - ephemeral containers. - type: string - securityContext: - description: |- - Optional: SecurityContext defines the security options the ephemeral container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to the container. - type: string - role: - description: Role is a SELinux role label that applies to the container. - type: string - type: - description: Type is a SELinux type label that applies to the container. - type: string - user: - description: User is a SELinux user label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: Probes are not allowed for ephemeral containers. - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - targetContainerName: - description: |- - If set, the name of the container from PodSpec that this ephemeral container targets. - The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. - If not set then the ephemeral container uses the namespaces configured in the Pod spec. - - The container runtime must implement support for this feature. If the runtime does not - support namespace targeting then the result of setting this field is undefined. - type: string - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. - items: - description: volumeDevice describes a mapping of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. - type: string - name: - description: name must match the name of a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. - Cannot be updated. - items: - description: VolumeMount describes a mounting of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - hostAliases: - description: |- - HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts - file if specified. - items: - description: |- - HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the - pod's hosts file. - properties: - hostnames: - description: Hostnames for the above IP address. - items: - type: string - type: array - x-kubernetes-list-type: atomic - ip: - description: IP address of the host file entry. - type: string - required: - - ip - type: object - type: array - x-kubernetes-list-map-keys: - - ip - x-kubernetes-list-type: map - hostIPC: - description: |- - Use the host's ipc namespace. - Optional: Default to false. - type: boolean - hostNetwork: - description: |- - Host networking requested for this pod. Use the host's network namespace. - If this option is set, the ports that will be used must be specified. - Default to false. - type: boolean - hostPID: - description: |- - Use the host's pid namespace. - Optional: Default to false. - type: boolean - hostUsers: - description: |- - Use the host's user namespace. - Optional: Default to true. - If set to true or not present, the pod will be run in the host user namespace, useful - for when the pod needs a feature only available to the host user namespace, such as - loading a kernel module with CAP_SYS_MODULE. - When set to false, a new userns is created for the pod. Setting false is useful for - mitigating container breakout vulnerabilities even allowing users to run their - containers as root without actually having root privileges on the host. - This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. - type: boolean - hostname: - description: |- - Specifies the hostname of the Pod - If not specified, the pod's hostname will be set to a system-defined value. - type: string - imagePullSecrets: - description: |- - ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. - If specified, these secrets will be passed to individual puller implementations for them to use. - More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod - items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - initContainers: - description: |- - List of initialization containers belonging to the pod. - Init containers are executed in order prior to containers being started. If any - init container fails, the pod is considered to have failed and is handled according - to its restartPolicy. The name for an init container or normal container must be - unique among all containers. - Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. - The resourceRequirements of an init container are taken into account during scheduling - by finding the highest request/limit for each resource type, and then using the max of - of that value or the sum of the normal containers. Limits are applied to init containers - in a similar fashion. - Init containers cannot currently be added or removed. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ - items: - description: A single application container that you want to run within a pod. - properties: - args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment variable present in a Container. - properties: - name: - description: Name of the environment variable. Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret in the pod's namespace - properties: - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration that the container should sleep. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration that the container should sleep. - properties: - seconds: - description: Seconds is the number of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. - type: string - ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. - type: string - securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to the container. - type: string - role: - description: Role is a SELinux role label that applies to the container. - type: string - type: - description: Type is a SELinux type label that applies to the container. - type: string - user: - description: User is a SELinux user label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection to a TCP port. - properties: - host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. - items: - description: volumeDevice describes a mapping of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. - type: string - name: - description: name must match the name of a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. - items: - description: VolumeMount describes a mounting of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - nodeName: - description: |- - NodeName indicates in which node this pod is scheduled. - If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. - Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. - This field should not be used to express a desire for the pod to be scheduled on a specific node. - https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename - type: string - nodeSelector: - additionalProperties: - type: string - description: |- - NodeSelector is a selector which must be true for the pod to fit on a node. - Selector which must match a node's labels for the pod to be scheduled on that node. - More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - type: object - x-kubernetes-map-type: atomic - os: - description: |- - Specifies the OS of the containers in the pod. - Some pod and container fields are restricted if this is set. - - If the OS field is set to linux, the following fields must be unset: - -securityContext.windowsOptions - - If the OS field is set to windows, following fields must be unset: - - spec.hostPID - - spec.hostIPC - - spec.hostUsers - - spec.securityContext.appArmorProfile - - spec.securityContext.seLinuxOptions - - spec.securityContext.seccompProfile - - spec.securityContext.fsGroup - - spec.securityContext.fsGroupChangePolicy - - spec.securityContext.sysctls - - spec.shareProcessNamespace - - spec.securityContext.runAsUser - - spec.securityContext.runAsGroup - - spec.securityContext.supplementalGroups - - spec.securityContext.supplementalGroupsPolicy - - spec.containers[*].securityContext.appArmorProfile - - spec.containers[*].securityContext.seLinuxOptions - - spec.containers[*].securityContext.seccompProfile - - spec.containers[*].securityContext.capabilities - - spec.containers[*].securityContext.readOnlyRootFilesystem - - spec.containers[*].securityContext.privileged - - spec.containers[*].securityContext.allowPrivilegeEscalation - - spec.containers[*].securityContext.procMount - - spec.containers[*].securityContext.runAsUser - - spec.containers[*].securityContext.runAsGroup - properties: - name: - description: |- - Name is the name of the operating system. The currently supported values are linux and windows. - Additional value may be defined in future and can be one of: - https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration - Clients should expect to handle additional values and treat unrecognized values in this field as os: null - type: string - required: - - name - type: object - overhead: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. - This field will be autopopulated at admission time by the RuntimeClass admission controller. If - the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. - The RuntimeClass admission controller will reject Pod create requests which have the overhead already - set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value - defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. - More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md - type: object - preemptionPolicy: - description: |- - PreemptionPolicy is the Policy for preempting pods with lower priority. - One of Never, PreemptLowerPriority. - Defaults to PreemptLowerPriority if unset. - type: string - priority: - description: |- - The priority value. Various system components use this field to find the - priority of the pod. When Priority Admission Controller is enabled, it - prevents users from setting this field. The admission controller populates - this field from PriorityClassName. - The higher the value, the higher the priority. - format: int32 - type: integer - priorityClassName: - description: |- - If specified, indicates the pod's priority. "system-node-critical" and - "system-cluster-critical" are two special keywords which indicate the - highest priorities with the former being the highest priority. Any other - name must be defined by creating a PriorityClass object with that name. - If not specified, the pod priority will be default or zero if there is no - default. - type: string - readinessGates: - description: |- - If specified, all readiness gates will be evaluated for pod readiness. - A pod is ready when all its containers are ready AND - all conditions specified in the readiness gates have status equal to "True" - More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates - items: - description: PodReadinessGate contains the reference to a pod condition - properties: - conditionType: - description: ConditionType refers to a condition in the pod's condition list with matching type. - type: string - required: - - conditionType - type: object - type: array - x-kubernetes-list-type: atomic - resourceClaims: - description: |- - ResourceClaims defines which ResourceClaims must be allocated - and reserved before the Pod is allowed to start. The resources - will be made available to those containers which consume them - by name. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. - items: - description: |- - PodResourceClaim references exactly one ResourceClaim, either directly - or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim - for the pod. - - It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. - Containers that need access to the ResourceClaim reference it with this name. - properties: - name: - description: |- - Name uniquely identifies this resource claim inside the pod. - This must be a DNS_LABEL. - type: string - resourceClaimName: - description: |- - ResourceClaimName is the name of a ResourceClaim object in the same - namespace as this pod. - - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. - type: string - resourceClaimTemplateName: - description: |- - ResourceClaimTemplateName is the name of a ResourceClaimTemplate - object in the same namespace as this pod. - - The template will be used to create a new ResourceClaim, which will - be bound to this pod. When this pod is deleted, the ResourceClaim - will also be deleted. The pod name and resource name, along with a - generated component, will be used to form a unique name for the - ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. - - This field is immutable and no changes will be made to the - corresponding ResourceClaim by the control plane after creating the - ResourceClaim. - - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - resources: - description: |- - Resources is the total amount of CPU and Memory resources required by all - containers in the pod. It supports specifying Requests and Limits for - "cpu" and "memory" resource names only. ResourceClaims are not supported. - - This field enables fine-grained control over resource allocation for the - entire pod, allowing resource sharing among containers in a pod. - - This is an alpha field and requires enabling the PodLevelResources feature - gate. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - Restart policy for all containers within the pod. - One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. - Default to Always. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy - type: string - runtimeClassName: - description: |- - RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used - to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. - If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an - empty definition that uses the default runtime handler. - More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class - type: string - schedulerName: - description: |- - If specified, the pod will be dispatched by specified scheduler. - If not specified, the pod will be dispatched by default scheduler. - type: string - schedulingGates: - description: |- - SchedulingGates is an opaque list of values that if specified will block scheduling the pod. - If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the - scheduler will not attempt to schedule the pod. - - SchedulingGates can only be set at pod creation time, and be removed only afterwards. - items: - description: PodSchedulingGate is associated to a Pod to guard its scheduling. - properties: - name: - description: |- - Name of the scheduling gate. - Each scheduling gate must have a unique name field. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - securityContext: - description: |- - SecurityContext holds pod-level security attributes and common container settings. - Optional: Defaults to empty. See type description for default values of each field. - properties: - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - fsGroup: - description: |- - A special supplemental group that applies to all containers in a pod. - Some volume types allow the Kubelet to change the ownership of that volume - to be owned by the pod: - - 1. The owning GID will be the FSGroup - 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) - 3. The permission bits are OR'd with rw-rw---- - - If unset, the Kubelet will not modify the ownership and permissions of any volume. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - fsGroupChangePolicy: - description: |- - fsGroupChangePolicy defines behavior of changing ownership and permission of the volume - before being exposed inside Pod. This field will only apply to - volume types which support fsGroup based ownership(and permissions). - It will have no effect on ephemeral volume types such as: secret, configmaps - and emptydir. - Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. - Note that this field cannot be set when spec.os.name is windows. - type: string - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxChangePolicy: - description: |- - seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. - It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. - Valid values are "MountOption" and "Recursive". - - "Recursive" means relabeling of all files on all Pod volumes by the container runtime. - This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. - - "MountOption" mounts all eligible Pod volumes with `-o context` mount option. - This requires all Pods that share the same volume to use the same SELinux label. - It is not possible to share the same volume among privileged and unprivileged Pods. - Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes - whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their - CSIDriver instance. Other volumes are always re-labelled recursively. - "MountOption" value is allowed only when SELinuxMount feature gate is enabled. - - If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. - If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes - and "Recursive" for all other volumes. - - This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. - - All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. - Note that this field cannot be set when spec.os.name is windows. - type: string - seLinuxOptions: - description: |- - The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in SecurityContext. If set in - both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies to the container. - type: string - role: - description: Role is a SELinux role label that applies to the container. - type: string - type: - description: Type is a SELinux type label that applies to the container. - type: string - user: - description: User is a SELinux user label that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - supplementalGroups: - description: |- - A list of groups applied to the first process run in each container, in - addition to the container's primary GID and fsGroup (if specified). If - the SupplementalGroupsPolicy feature is enabled, the - supplementalGroupsPolicy field determines whether these are in addition - to or instead of any group memberships defined in the container image. - If unspecified, no additional groups are added, though group memberships - defined in the container image may still be used, depending on the - supplementalGroupsPolicy field. - Note that this field cannot be set when spec.os.name is windows. - items: - format: int64 - type: integer - type: array - x-kubernetes-list-type: atomic - supplementalGroupsPolicy: - description: |- - Defines how supplemental groups of the first container processes are calculated. - Valid values are "Merge" and "Strict". If not specified, "Merge" is used. - (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled - and the container runtime must implement support for this feature. - Note that this field cannot be set when spec.os.name is windows. - type: string - sysctls: - description: |- - Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported - sysctls (by the container runtime) might fail to launch. - Note that this field cannot be set when spec.os.name is windows. - items: - description: Sysctl defines a kernel parameter to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options within a container's SecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - serviceAccount: - description: |- - DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. - Deprecated: Use serviceAccountName instead. - type: string - serviceAccountName: - description: |- - ServiceAccountName is the name of the ServiceAccount to use to run this pod. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ - type: string - setHostnameAsFQDN: - description: |- - If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). - In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). - In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. - If a pod does not have FQDN, this has no effect. - Default to false. - type: boolean - shareProcessNamespace: - description: |- - Share a single process namespace between all of the containers in a pod. - When this is set containers will be able to view and signal processes from other containers - in the same pod, and the first process in each container will not be assigned PID 1. - HostPID and ShareProcessNamespace cannot both be set. - Optional: Default to false. - type: boolean - subdomain: - description: |- - If specified, the fully qualified Pod hostname will be "...svc.". - If not specified, the pod will not have a domainname at all. - type: string - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - If this value is nil, the default grace period will be used instead. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - Defaults to 30 seconds. - format: int64 - type: integer - tolerations: - description: If specified, the pod's tolerations. - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - topologySpreadConstraints: - description: |- - TopologySpreadConstraints describes how a group of pods ought to spread across topology - domains. Scheduler will schedule pods in a way which abides by the constraints. - All topologySpreadConstraints are ANDed. - items: - description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. - properties: - labelSelector: - description: |- - LabelSelector is used to find matching pods. - Pods that match this label selector are counted to determine the number of pods - in their corresponding topology domain. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select the pods over which - spreading will be calculated. The keys are used to lookup values from the - incoming pod labels, those key-value labels are ANDed with labelSelector - to select the group of existing pods over which spreading will be calculated - for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - MatchLabelKeys cannot be set when LabelSelector isn't set. - Keys that don't exist in the incoming pod labels will - be ignored. A null or empty list means only match against labelSelector. - - This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxSkew: - description: |- - MaxSkew describes the degree to which pods may be unevenly distributed. - When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference - between the number of matching pods in the target topology and the global minimum. - The global minimum is the minimum number of matching pods in an eligible domain - or zero if the number of eligible domains is less than MinDomains. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 2/2/1: - In this case, the global minimum is 1. - | zone1 | zone2 | zone3 | - | P P | P P | P | - - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; - scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) - violate MaxSkew(1). - - if MaxSkew is 2, incoming pod can be scheduled onto any zone. - When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence - to topologies that satisfy it. - It's a required field. Default value is 1 and 0 is not allowed. - format: int32 - type: integer - minDomains: - description: |- - MinDomains indicates a minimum number of eligible domains. - When the number of eligible domains with matching topology keys is less than minDomains, - Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. - And when the number of eligible domains with matching topology keys equals or greater than minDomains, - this value has no effect on scheduling. - As a result, when the number of eligible domains is less than minDomains, - scheduler won't schedule more than maxSkew Pods to those domains. - If value is nil, the constraint behaves as if MinDomains is equal to 1. - Valid values are integers greater than 0. - When value is not nil, WhenUnsatisfiable must be DoNotSchedule. - - For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same - labelSelector spread as 2/2/2: - | zone1 | zone2 | zone3 | - | P P | P P | P P | - The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. - In this situation, new pod with the same labelSelector cannot be scheduled, - because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, - it will violate MaxSkew. - format: int32 - type: integer - nodeAffinityPolicy: - description: |- - NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector - when calculating pod topology spread skew. Options are: - - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. - - If this value is nil, the behavior is equivalent to the Honor policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. - type: string - nodeTaintsPolicy: - description: |- - NodeTaintsPolicy indicates how we will treat node taints when calculating - pod topology spread skew. Options are: - - Honor: nodes without taints, along with tainted nodes for which the incoming pod - has a toleration, are included. - - Ignore: node taints are ignored. All nodes are included. - - If this value is nil, the behavior is equivalent to the Ignore policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. - type: string - topologyKey: - description: |- - TopologyKey is the key of node labels. Nodes that have a label with this key - and identical values are considered to be in the same topology. - We consider each as a "bucket", and try to put balanced number - of pods into each bucket. - We define a domain as a particular instance of a topology. - Also, we define an eligible domain as a domain whose nodes meet the requirements of - nodeAffinityPolicy and nodeTaintsPolicy. - e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. - And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. - It's a required field. - type: string - whenUnsatisfiable: - description: |- - WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy - the spread constraint. - - DoNotSchedule (default) tells the scheduler not to schedule it. - - ScheduleAnyway tells the scheduler to schedule the pod in any location, - but giving higher precedence to topologies that would help reduce the - skew. - A constraint is considered "Unsatisfiable" for an incoming pod - if and only if every possible node assignment for that pod would violate - "MaxSkew" on some topology. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 3/1/1: - | zone1 | zone2 | zone3 | - | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled - to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies - MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler - won't make it *more* imbalanced. - It's a required field. - type: string - required: - - maxSkew - - topologyKey - - whenUnsatisfiable - type: object - type: array - x-kubernetes-list-map-keys: - - topologyKey - - whenUnsatisfiable - x-kubernetes-list-type: map - volumes: - description: |- - List of volumes that can be mounted by containers belonging to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes - items: - description: Volume represents a named volume in a pod that may be accessed by any container in the pod. - properties: - awsElasticBlockStore: - description: |- - awsElasticBlockStore represents an AWS Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - Deprecated: AWSElasticBlockStore is deprecated. All operations for the in-tree - awsElasticBlockStore type are redirected to the ebs.csi.aws.com CSI driver. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - format: int32 - type: integer - readOnly: - description: |- - readOnly value true will force the readOnly setting in VolumeMounts. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: boolean - volumeID: - description: |- - volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: string - required: - - volumeID - type: object - azureDisk: - description: |- - azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. - Deprecated: AzureDisk is deprecated. All operations for the in-tree azureDisk type - are redirected to the disk.csi.azure.com CSI driver. - properties: - cachingMode: - description: 'cachingMode is the Host Caching mode: None, Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data disk in the blob storage - type: string - diskURI: - description: diskURI is the URI of data disk in the blob storage - type: string - fsType: - default: ext4 - description: |- - fsType is Filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - kind: - description: 'kind expected values are Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared' - type: string - readOnly: - default: false - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: |- - azureFile represents an Azure File Service mount on the host and bind mount to the pod. - Deprecated: AzureFile is deprecated. All operations for the in-tree azureFile type - are redirected to the file.csi.azure.com CSI driver. - properties: - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret that contains Azure Storage Account Name and Key - type: string - shareName: - description: shareName is the azure share Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: |- - cephFS represents a Ceph FS mount on the host that shares a pod's lifetime. - Deprecated: CephFS is deprecated and the in-tree cephfs type is no longer supported. - properties: - monitors: - description: |- - monitors is Required: Monitors is a collection of Ceph monitors - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - items: - type: string - type: array - x-kubernetes-list-type: atomic - path: - description: 'path is Optional: Used as the mounted root, rather than the full Ceph tree, default is /' - type: string - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: boolean - secretFile: - description: |- - secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - secretRef: - description: |- - secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - user: - description: |- - user is optional: User is the rados user name, default is admin - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - required: - - monitors - type: object - cinder: - description: |- - cinder represents a cinder volume attached and mounted on kubelets host machine. - Deprecated: Cinder is deprecated. All operations for the in-tree cinder type - are redirected to the cinder.csi.openstack.org CSI driver. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: boolean - secretRef: - description: |- - secretRef is optional: points to a secret object containing parameters used to connect - to OpenStack. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - volumeID: - description: |- - volumeID used to identify the volume in cinder. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap that should populate this volume - properties: - defaultMode: - description: |- - defaultMode is optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional specify whether the ConfigMap or its keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - csi: - description: csi (Container Storage Interface) represents ephemeral storage that is handled by certain external CSI drivers. - properties: - driver: - description: |- - driver is the name of the CSI driver that handles this volume. - Consult with your admin for the correct name as registered in the cluster. - type: string - fsType: - description: |- - fsType to mount. Ex. "ext4", "xfs", "ntfs". - If not provided, the empty value is passed to the associated CSI driver - which will determine the default filesystem to apply. - type: string - nodePublishSecretRef: - description: |- - nodePublishSecretRef is a reference to the secret object containing - sensitive information to pass to the CSI driver to complete the CSI - NodePublishVolume and NodeUnpublishVolume calls. - This field is optional, and may be empty if no secret is required. If the - secret object contains more than one secret, all secret references are passed. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - readOnly: - description: |- - readOnly specifies a read-only configuration for the volume. - Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: |- - volumeAttributes stores driver-specific properties that are passed to the CSI - driver. Consult your driver's documentation for supported values. - type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API about the pod that should populate this volume - properties: - defaultMode: - description: |- - Optional: mode bits to use on created files by default. Must be a - Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: Items is a list of downward API volume file - items: - description: DownwardAPIVolumeFile represents information to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - emptyDir: - description: |- - emptyDir represents a temporary directory that shares a pod's lifetime. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - properties: - medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: |- - ephemeral represents a volume that is handled by a cluster storage driver. - The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, - and deleted when the pod is removed. - - Use this if: - a) the volume is only needed while the pod runs, - b) features of normal volumes like restoring from snapshot or capacity - tracking are needed, - c) the storage driver is specified through a storage class, and - d) the storage driver supports dynamic volume provisioning through - a PersistentVolumeClaim (see EphemeralVolumeSource for more - information on the connection between this volume type - and PersistentVolumeClaim). - - Use PersistentVolumeClaim or one of the vendor-specific - APIs for volumes that persist for longer than the lifecycle - of an individual pod. - - Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to - be used that way - see the documentation of the driver for - more information. - - A pod can use both types of ephemeral volumes and - persistent volumes at the same time. - properties: - volumeClaimTemplate: - description: |- - Will be used to create a stand-alone PVC to provision the volume. - The pod in which this EphemeralVolumeSource is embedded will be the - owner of the PVC, i.e. the PVC will be deleted together with the - pod. The name of the PVC will be `-` where - `` is the name from the `PodSpec.Volumes` array - entry. Pod validation will reject the pod if the concatenated name - is not valid for a PVC (for example, too long). - - An existing PVC with that name that is not owned by the pod - will *not* be used for the pod to avoid using an unrelated - volume by mistake. Starting the pod is then blocked until - the unrelated PVC is removed. If such a pre-created PVC is - meant to be used by the pod, the PVC has to updated with an - owner reference to the pod once the pod exists. Normally - this should not be necessary, but it may be useful when - manually reconstructing a broken cluster. - - This field is read-only and no changes will be made by Kubernetes - to the PVC after it has been created. - - Required, must not be nil. - properties: - metadata: - description: |- - May contain labels and annotations that will be copied into the PVC - when creating it. No other fields are allowed and will be rejected during - validation. - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - The specification for the PersistentVolumeClaim. The entire content is - copied unchanged into the PVC that gets created from this - template. The same fields as in a PersistentVolumeClaim - are also valid here. - properties: - accessModes: - description: |- - accessModes contains the desired access modes the volume should have. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 - items: - type: string - type: array - x-kubernetes-list-type: atomic - dataSource: - description: |- - dataSource field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - If the provisioner or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified data source. - When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, - and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. - If the namespace is specified, then dataSourceRef will not be copied to dataSource. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: |- - dataSourceRef specifies the object from which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a non-empty API group (non - core object) or a PersistentVolumeClaim object. - When this field is specified, volume binding will only succeed if the type of - the specified object matches some installed volume populator or dynamic - provisioner. - This field will replace the functionality of the dataSource field and as such - if both fields are non-empty, they must have the same value. For backwards - compatibility, when namespace isn't specified in dataSourceRef, - both fields (dataSource and dataSourceRef) will be set to the same - value automatically if one of them is empty and the other is non-empty. - When namespace is specified in dataSourceRef, - dataSource isn't set to the same value and must be empty. - There are three important differences between dataSource and dataSourceRef: - * While dataSource only allows two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping them), dataSourceRef - preserves all values, and generates an error if a disallowed value is - specified. - * While dataSource only allows local objects, dataSourceRef allows objects - in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. - (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - namespace: - description: |- - Namespace is the namespace of resource being referenced - Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. - (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: |- - resources represents the minimum resources the volume should have. - If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements - that are lower than previous value but must still be higher than capacity recorded in the - status field of the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - description: selector is a label query over volumes to consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: |- - storageClassName is the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 - type: string - volumeAttributesClassName: - description: |- - volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. - If specified, the CSI driver will create or update the volume with the attributes defined - in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. - If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be - set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource - exists. - More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). - type: string - volumeMode: - description: |- - volumeMode defines what type of volume is required by the claim. - Value of Filesystem is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference to the PersistentVolume backing this claim. - type: string - type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - lun: - description: 'lun is Optional: FC target lun number' - format: int32 - type: integer - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target worldwide names (WWNs)' - items: - type: string - type: array - x-kubernetes-list-type: atomic - wwids: - description: |- - wwids Optional: FC volume world wide identifiers (wwids) - Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - flexVolume: - description: |- - flexVolume represents a generic volume resource that is - provisioned/attached using an exec based plugin. - Deprecated: FlexVolume is deprecated. Consider using a CSIDriver instead. - properties: - driver: - description: driver is the name of the driver to use for this volume. - type: string - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field holds extra command options if any.' - type: object - readOnly: - description: |- - readOnly is Optional: defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef is Optional: secretRef is reference to the secret object containing - sensitive information to pass to the plugin scripts. This may be - empty if no secret object is specified. If the secret object - contains more than one secret, all secrets are passed to the plugin - scripts. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - required: - - driver - type: object - flocker: - description: |- - flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running. - Deprecated: Flocker is deprecated and the in-tree flocker type is no longer supported. - properties: - datasetName: - description: |- - datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker - should be considered as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the dataset. This is unique identifier of a Flocker dataset - type: string - type: object - gcePersistentDisk: - description: |- - gcePersistentDisk represents a GCE Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - Deprecated: GCEPersistentDisk is deprecated. All operations for the in-tree - gcePersistentDisk type are redirected to the pd.csi.storage.gke.io CSI driver. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - properties: - fsType: - description: |- - fsType is filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - format: int32 - type: integer - pdName: - description: |- - pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: boolean - required: - - pdName - type: object - gitRepo: - description: |- - gitRepo represents a git repository at a particular revision. - Deprecated: GitRepo is deprecated. To provision a container with a git repo, mount an - EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir - into the Pod's container. - properties: - directory: - description: |- - directory is the target directory name. - Must not contain or start with '..'. If '.' is supplied, the volume directory will be the - git repository. Otherwise, if specified, the volume will contain the git repository in - the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for the specified revision. - type: string - required: - - repository - type: object - glusterfs: - description: |- - glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. - Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - More info: https://examples.k8s.io/volumes/glusterfs/README.md - properties: - endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: string - path: - description: |- - path is the Glusterfs volume path. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: string - readOnly: - description: |- - readOnly here will force the Glusterfs volume to be mounted with read-only permissions. - Defaults to false. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: boolean - required: - - endpoints - - path - type: object - hostPath: - description: |- - hostPath represents a pre-existing file or directory on the host - machine that is directly exposed to the container. This is generally - used for system agents or other privileged things that are allowed - to see the host machine. Most containers will NOT need this. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - properties: - path: - description: |- - path of the directory on the host. - If the path is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - type: - description: |- - type for HostPath Volume - Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - required: - - path - type: object - image: - description: |- - image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. - The volume is resolved at pod startup depending on which PullPolicy value is provided: - - - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - - The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. - A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. - The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. - The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. - The volume will be mounted read-only (ro) and non-executable files (noexec). - Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). - The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. - properties: - pullPolicy: - description: |- - Policy for pulling OCI objects. Possible values are: - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - type: string - reference: - description: |- - Required: Image or artifact reference to be used. - Behaves in the same way as pod.spec.containers[*].image. - Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - type: object - iscsi: - description: |- - iscsi represents an ISCSI Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md - properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether support iSCSI Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether support iSCSI Session CHAP authentication - type: boolean - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - type: string - initiatorName: - description: |- - initiatorName is the custom iSCSI Initiator Name. - If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface - : will be created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified Name. - type: string - iscsiInterface: - default: default - description: |- - iscsiInterface is the interface Name that uses an iSCSI transport. - Defaults to 'default' (tcp). - type: string - lun: - description: lun represents iSCSI Target Lun number. - format: int32 - type: integer - portals: - description: |- - portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - items: - type: string - type: array - x-kubernetes-list-type: atomic - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret for iSCSI target and initiator authentication - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - targetPortal: - description: |- - targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - description: |- - name of the volume. - Must be a DNS_LABEL and unique within the pod. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - nfs: - description: |- - nfs represents an NFS mount on the host that shares a pod's lifetime - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - properties: - path: - description: |- - path that is exported by the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - readOnly: - description: |- - readOnly here will force the NFS export to be mounted with read-only permissions. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: boolean - server: - description: |- - server is the hostname or IP address of the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: |- - persistentVolumeClaimVolumeSource represents a reference to a - PersistentVolumeClaim in the same namespace. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - properties: - claimName: - description: |- - claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - type: string - readOnly: - description: |- - readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: |- - photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine. - Deprecated: PhotonPersistentDisk is deprecated and the in-tree photonPersistentDisk type is no longer supported. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - pdID: - description: pdID is the ID that identifies Photon Controller persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: |- - portworxVolume represents a portworx volume attached and mounted on kubelets host machine. - Deprecated: PortworxVolume is deprecated. All operations for the in-tree portworxVolume type - are redirected to the pxd.portworx.com CSI driver when the CSIMigrationPortworx feature-gate - is on. - properties: - fsType: - description: |- - fSType represents the filesystem type to mount - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources secrets, configmaps, and downward API - properties: - defaultMode: - description: |- - defaultMode are the mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - sources: - description: |- - sources is the list of volume projections. Each entry in this list - handles one source. - items: - description: |- - Projection that may be projected along with other supported volume types. - Exactly one of these fields must be set. - properties: - clusterTrustBundle: - description: |- - ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field - of ClusterTrustBundle objects in an auto-updating file. - - Alpha, gated by the ClusterTrustBundleProjection feature gate. - - ClusterTrustBundle objects can either be selected by name, or by the - combination of signer name and a label selector. - - Kubelet performs aggressive normalization of the PEM contents written - into the pod filesystem. Esoteric PEM features such as inter-block - comments and block headers are stripped. Certificates are deduplicated. - The ordering of certificates within the file is arbitrary, and Kubelet - may change the order over time. - properties: - labelSelector: - description: |- - Select all ClusterTrustBundles that match this label selector. Only has - effect if signerName is set. Mutually-exclusive with name. If unset, - interpreted as "match nothing". If set but empty, interpreted as "match - everything". - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - name: - description: |- - Select a single ClusterTrustBundle by object name. Mutually-exclusive - with signerName and labelSelector. - type: string - optional: - description: |- - If true, don't block pod startup if the referenced ClusterTrustBundle(s) - aren't available. If using name, then the named ClusterTrustBundle is - allowed not to exist. If using signerName, then the combination of - signerName and labelSelector is allowed to match zero - ClusterTrustBundles. - type: boolean - path: - description: Relative path from the volume root to write the bundle. - type: string - signerName: - description: |- - Select all ClusterTrustBundles that match this signer name. - Mutually-exclusive with name. The contents of all selected - ClusterTrustBundles will be unified and deduplicated. - type: string - required: - - path - type: object - configMap: - description: configMap information about the configMap data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional specify whether the ConfigMap or its keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - downwardAPI: - description: downwardAPI information about the downwardAPI data to project - properties: - items: - description: Items is a list of DownwardAPIVolume file - items: - description: DownwardAPIVolumeFile represents information to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.' - properties: - apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - secret: - description: secret information about the secret data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional field specify whether the Secret or its key must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - serviceAccountToken: - description: serviceAccountToken is information about the serviceAccountToken data to project - properties: - audience: - description: |- - audience is the intended audience of the token. A recipient of a token - must identify itself with an identifier specified in the audience of the - token, and otherwise should reject the token. The audience defaults to the - identifier of the apiserver. - type: string - expirationSeconds: - description: |- - expirationSeconds is the requested duration of validity of the service - account token. As the token approaches expiration, the kubelet volume - plugin will proactively rotate the service account token. The kubelet will - start trying to rotate the token if the token is older than 80 percent of - its time to live or if the token is older than 24 hours.Defaults to 1 hour - and must be at least 10 minutes. - format: int64 - type: integer - path: - description: |- - path is the path relative to the mount point of the file to project the - token into. - type: string - required: - - path - type: object - type: object - type: array - x-kubernetes-list-type: atomic - type: object - quobyte: - description: |- - quobyte represents a Quobyte mount on the host that shares a pod's lifetime. - Deprecated: Quobyte is deprecated and the in-tree quobyte type is no longer supported. - properties: - group: - description: |- - group to map volume access to - Default is no group - type: string - readOnly: - description: |- - readOnly here will force the Quobyte volume to be mounted with read-only permissions. - Defaults to false. - type: boolean - registry: - description: |- - registry represents a single or multiple Quobyte Registry services - specified as a string as host:port pair (multiple entries are separated with commas) - which acts as the central registry for volumes - type: string - tenant: - description: |- - tenant owning the given Quobyte volume in the Backend - Used with dynamically provisioned Quobyte volumes, value is set by the plugin - type: string - user: - description: |- - user to map volume access to - Defaults to serivceaccount user - type: string - volume: - description: volume is a string that references an already created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: |- - rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. - Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - More info: https://examples.k8s.io/volumes/rbd/README.md - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - type: string - image: - description: |- - image is the rados image name. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - keyring: - default: /etc/ceph/keyring - description: |- - keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - monitors: - description: |- - monitors is a collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - items: - type: string - type: array - x-kubernetes-list-type: atomic - pool: - default: rbd - description: |- - pool is the rados pool name. - Default is rbd. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: boolean - secretRef: - description: |- - secretRef is name of the authentication secret for RBDUser. If provided - overrides keyring. - Default is nil. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - user: - default: admin - description: |- - user is the rados user name. - Default is admin. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - required: - - image - - monitors - type: object - scaleIO: - description: |- - scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. - Deprecated: ScaleIO is deprecated and the in-tree scaleIO type is no longer supported. - properties: - fsType: - default: xfs - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". - Default is "xfs". - type: string - gateway: - description: gateway is the host address of the ScaleIO API Gateway. - type: string - protectionDomain: - description: protectionDomain is the name of the ScaleIO Protection Domain for the configured storage. - type: string - readOnly: - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef references to the secret for ScaleIO user and other - sensitive information. If this is not provided, Login operation will fail. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - sslEnabled: - description: sslEnabled Flag enable/disable SSL communication with Gateway, default false - type: boolean - storageMode: - default: ThinProvisioned - description: |- - storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage Pool associated with the protection domain. - type: string - system: - description: system is the name of the storage system as configured in ScaleIO. - type: string - volumeName: - description: |- - volumeName is the name of a volume already created in the ScaleIO system - that is associated with this volume source. - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - description: |- - secret represents a secret that should populate this volume. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - properties: - defaultMode: - description: |- - defaultMode is Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values - for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items If unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - optional: - description: optional field specify whether the Secret or its keys must be defined - type: boolean - secretName: - description: |- - secretName is the name of the secret in the pod's namespace to use. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - type: string - type: object - storageos: - description: |- - storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. - Deprecated: StorageOS is deprecated and the in-tree storageos type is no longer supported. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef specifies the secret to use for obtaining the StorageOS API - credentials. If not specified, default values will be attempted. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - volumeName: - description: |- - volumeName is the human-readable name of the StorageOS volume. Volume - names are only unique within a namespace. - type: string - volumeNamespace: - description: |- - volumeNamespace specifies the scope of the volume within StorageOS. If no - namespace is specified then the Pod's namespace will be used. This allows the - Kubernetes name scoping to be mirrored within StorageOS for tighter integration. - Set VolumeName to any name to override the default behaviour. - Set to "default" if you are not using namespaces within StorageOS. - Namespaces that do not pre-exist within StorageOS will be created. - type: string - type: object - vsphereVolume: - description: |- - vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine. - Deprecated: VsphereVolume is deprecated. All operations for the in-tree vsphereVolume type - are redirected to the csi.vsphere.vmware.com CSI driver. - properties: - fsType: - description: |- - fsType is filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName. - type: string - storagePolicyName: - description: storagePolicyName is the storage Policy Based Management (SPBM) profile name. - type: string - volumePath: - description: volumePath is the path that identifies vSphere volume vmdk - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - containers - type: object - type: object - ttlSecondsAfterFinished: - description: |- - ttlSecondsAfterFinished limits the lifetime of a Job that has finished - execution (either Complete or Failed). If this field is set, - ttlSecondsAfterFinished after the Job finishes, it is eligible to be - automatically deleted. When the Job is being deleted, its lifecycle - guarantees (e.g. finalizers) will be honored. If this field is unset, - the Job won't be automatically deleted. If this field is set to zero, - the Job becomes eligible to be deleted immediately after it finishes. - format: int32 - type: integer - required: - - template - type: object - type: object - required: - - name - - template - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - startupPolicy: - description: |- - StartupPolicy, if set, configures in what order jobs must be started - Deprecated: StartupPolicy is deprecated, please use the DependsOn API. - properties: - startupPolicyOrder: - description: |- - StartupPolicyOrder determines the startup order of the ReplicatedJobs. - AnyOrder means to start replicated jobs in any order. - InOrder means to start them as they are listed in the JobSet. A ReplicatedJob is started only - when all the jobs of the previous one are ready. - enum: - - AnyOrder - - InOrder - type: string - required: - - startupPolicyOrder - type: object - x-kubernetes-validations: - - message: Value is immutable - rule: self == oldSelf - successPolicy: - description: |- - SuccessPolicy configures when to declare the JobSet as - succeeded. - The JobSet is always declared succeeded if all jobs in the set - finished with status complete. - properties: - operator: - description: Operator determines either All or Any of the selected jobs should succeed to consider the JobSet successful - enum: - - All - - Any - type: string - targetReplicatedJobs: - description: |- - TargetReplicatedJobs are the names of the replicated jobs the operator will apply to. - A null or empty list will apply to all replicatedJobs. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - operator - type: object - x-kubernetes-validations: - - message: Value is immutable - rule: self == oldSelf - suspend: - description: Suspend suspends all running child Jobs when set to true. - type: boolean - ttlSecondsAfterFinished: - description: |- - TTLSecondsAfterFinished limits the lifetime of a JobSet that has finished - execution (either Complete or Failed). If this field is set, - TTLSecondsAfterFinished after the JobSet finishes, it is eligible to be - automatically deleted. When the JobSet is being deleted, its lifecycle - guarantees (e.g. finalizers) will be honored. If this field is unset, - the JobSet won't be automatically deleted. If this field is set to zero, - the JobSet becomes eligible to be deleted immediately after it finishes. - format: int32 - minimum: 0 - type: integer - type: object - x-kubernetes-validations: - - message: StartupPolicy and DependsOn APIs are mutually exclusive - rule: '!(has(self.startupPolicy) && self.startupPolicy.startupPolicyOrder == ''InOrder'' && self.replicatedJobs.exists(x, has(x.dependsOn)))' - status: - description: JobSetStatus defines the observed state of JobSet - properties: - conditions: - items: - description: Condition contains details for one aspect of the current state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - replicatedJobsStatus: - description: ReplicatedJobsStatus track the number of JobsReady for each replicatedJob. - items: - description: ReplicatedJobStatus defines the observed ReplicatedJobs Readiness. - properties: - active: - description: |- - Active is the number of child Jobs with at least 1 pod in a running or pending state - which are not marked for deletion. - format: int32 - type: integer - failed: - description: Failed is the number of failed child Jobs. - format: int32 - type: integer - name: - description: Name of the ReplicatedJob. - type: string - ready: - description: |- - Ready is the number of child Jobs where the number of ready pods and completed pods - is greater than or equal to the total expected pod count for the Job (i.e., the minimum - of job.spec.parallelism and job.spec.completions). - format: int32 - type: integer - succeeded: - description: Succeeded is the number of successfully completed child Jobs. - format: int32 - type: integer - suspended: - description: Suspended is the number of child Jobs which are in a suspended state. - format: int32 - type: integer - required: - - active - - failed - - name - - ready - - succeeded - - suspended - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - restarts: - description: Restarts tracks the number of times the JobSet has restarted (i.e. recreated in case of RecreateAll policy). - format: int32 - type: integer - restartsCountTowardsMax: - description: RestartsCountTowardsMax tracks the number of times the JobSet has restarted that counts towards the maximum allowed number of restarts. - format: int32 - type: integer - terminalState: - description: |- - TerminalState the state of the JobSet when it finishes execution. - It can be either Complete or Failed. Otherwise, it is empty by default. - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: controller-manager - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: serviceaccount - app.kubernetes.io/part-of: jobset - name: jobset-controller-manager - namespace: jobset-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: leader-election-role - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: role - app.kubernetes.io/part-of: jobset - name: jobset-leader-election-role - namespace: jobset-system -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: jobset-manager-role -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - pods - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - update - - watch -- apiGroups: - - admissionregistration.k8s.io - resources: - - mutatingwebhookconfigurations - - validatingwebhookconfigurations - verbs: - - get - - list - - update - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs/status - verbs: - - get - - patch - - update -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/finalizers - verbs: - - update -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/status - verbs: - - get - - patch - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: metrics-reader - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: clusterrole - app.kubernetes.io/part-of: jobset - name: jobset-metrics-reader -rules: -- nonResourceURLs: - - /metrics - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: proxy-role - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: clusterrole - app.kubernetes.io/part-of: jobset - name: jobset-proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: leader-election-rolebinding - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: rolebinding - app.kubernetes.io/part-of: jobset - name: jobset-leader-election-rolebinding - namespace: jobset-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: jobset-leader-election-role -subjects: -- kind: ServiceAccount - name: jobset-controller-manager - namespace: jobset-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: manager-rolebinding - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: clusterrolebinding - app.kubernetes.io/part-of: jobset - name: jobset-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: jobset-manager-role -subjects: -- kind: ServiceAccount - name: jobset-controller-manager - namespace: jobset-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: jobset-metrics-reader-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: jobset-metrics-reader -subjects: -- kind: ServiceAccount - name: jobset-controller-manager - namespace: jobset-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: proxy-rolebinding - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: clusterrolebinding - app.kubernetes.io/part-of: jobset - name: jobset-proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: jobset-proxy-role -subjects: -- kind: ServiceAccount - name: jobset-controller-manager - namespace: jobset-system ---- -apiVersion: v1 -data: - controller_manager_config.yaml: |2 - - apiVersion: config.jobset.x-k8s.io/v1alpha1 - kind: Configuration - leaderElection: - leaderElect: true -kind: ConfigMap -metadata: - name: jobset-manager-config - namespace: jobset-system ---- -apiVersion: v1 -kind: Secret -metadata: - name: jobset-webhook-server-cert - namespace: jobset-system ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: jobset - control-plane: controller-manager - name: jobset-controller-manager-metrics-service - namespace: jobset-system -spec: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: 8443 - selector: - control-plane: controller-manager ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: webhook - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: webhook-service - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: service - app.kubernetes.io/part-of: jobset - name: jobset-webhook-service - namespace: jobset-system -spec: - ports: - - port: 443 - protocol: TCP - targetPort: 9443 - selector: - control-plane: controller-manager ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app.kubernetes.io/component: manager - app.kubernetes.io/created-by: jobset - app.kubernetes.io/instance: controller-manager - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: deployment - app.kubernetes.io/part-of: jobset - control-plane: controller-manager - name: jobset-controller-manager - namespace: jobset-system -spec: - replicas: 1 - selector: - matchLabels: - control-plane: controller-manager - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: manager - labels: - control-plane: controller-manager - spec: - containers: - - args: - - --config=/controller_manager_config.yaml - - --zap-log-level=2 - command: - - /manager - image: registry.k8s.io/jobset/jobset:v0.8.1 - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - ports: - - containerPort: 8443 - name: metrics - protocol: TCP - - containerPort: 9443 - name: webhook-server - protocol: TCP - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - memory: 4Gi - requests: - cpu: 500m - memory: 128Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - volumeMounts: - - mountPath: /controller_manager_config.yaml - name: manager-config - subPath: controller_manager_config.yaml - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true - securityContext: - runAsNonRoot: true - serviceAccountName: jobset-controller-manager - terminationGracePeriodSeconds: 10 - volumes: - - configMap: - name: jobset-manager-config - name: manager-config - - name: cert - secret: - defaultMode: 420 - secretName: jobset-webhook-server-cert - tolerations: - - effect: NoSchedule - key: components.gke.io/gke-managed-components - operator: Equal - value: "true" ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: jobset-mutating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: jobset-webhook-service - namespace: jobset-system - path: /mutate-jobset-x-k8s-io-v1alpha2-jobset - failurePolicy: Fail - name: mjobset.kb.io - rules: - - apiGroups: - - jobset.x-k8s.io - apiVersions: - - v1alpha2 - operations: - - CREATE - - UPDATE - resources: - - jobsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: jobset-webhook-service - namespace: jobset-system - path: /mutate--v1-pod - failurePolicy: Fail - name: mpod.kb.io - objectSelector: - matchExpressions: - - key: jobset.sigs.k8s.io/jobset-name - operator: Exists - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods - sideEffects: None ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: jobset-validating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: jobset-webhook-service - namespace: jobset-system - path: /validate-jobset-x-k8s-io-v1alpha2-jobset - failurePolicy: Fail - name: vjobset.kb.io - rules: - - apiGroups: - - jobset.x-k8s.io - apiVersions: - - v1alpha2 - operations: - - CREATE - - UPDATE - resources: - - jobsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: jobset-webhook-service - namespace: jobset-system - path: /validate--v1-pod - failurePolicy: Fail - name: vpod.kb.io - objectSelector: - matchExpressions: - - key: jobset.sigs.k8s.io/jobset-name - operator: Exists - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods - sideEffects: None diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.10.0.yaml b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.10.0.yaml deleted file mode 100644 index 8fb5db3638..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.10.0.yaml +++ /dev/null @@ -1,13185 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Namespace -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-system ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: admissionchecks.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: AdmissionCheck - listKind: AdmissionCheckList - plural: admissionchecks - singular: admissioncheck - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: AdmissionCheck is the Schema for the admissionchecks API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: AdmissionCheckSpec defines the desired state of AdmissionCheck - properties: - controllerName: - description: |- - controllerName identifies the controller that processes the AdmissionCheck, - not necessarily a Kubernetes Pod or Deployment name. Cannot be empty. - type: string - x-kubernetes-validations: - - message: field is immutable - rule: self == oldSelf - parameters: - description: |- - Parameters identifies a configuration with additional parameters for the - check. - properties: - apiGroup: - description: ApiGroup is the group for the resource being referenced. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: Kind is the type of the resource being referenced. - maxLength: 63 - pattern: ^(?i)[a-z]([-a-z0-9]*[a-z0-9])?$ - type: string - name: - description: Name is the name of the resource being referenced. - maxLength: 63 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - apiGroup - - kind - - name - type: object - retryDelayMinutes: - default: 15 - description: |- - RetryDelayMinutes specifies how long to keep the workload suspended after - a failed check (after it transitioned to False). When the delay period has passed, the check - state goes to "Unknown". The default is 15 min. - Deprecated: retryDelayMinutes has already been deprecated since v0.8 and will be removed in v1beta2. - format: int64 - type: integer - required: - - controllerName - type: object - status: - description: AdmissionCheckStatus defines the observed state of AdmissionCheck - properties: - conditions: - description: |- - conditions hold the latest available observations of the AdmissionCheck - current state. - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: clusterqueues.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: ClusterQueue - listKind: ClusterQueueList - plural: clusterqueues - shortNames: - - cq - singular: clusterqueue - scope: Cluster - versions: - - additionalPrinterColumns: - - description: Cohort that this ClusterQueue belongs to - jsonPath: .spec.cohort - name: Cohort - type: string - - description: The queueing strategy used to prioritize workloads - jsonPath: .spec.queueingStrategy - name: Strategy - priority: 1 - type: string - - description: Number of pending workloads - jsonPath: .status.pendingWorkloads - name: Pending Workloads - type: integer - - description: Number of admitted workloads that haven't finished yet - jsonPath: .status.admittedWorkloads - name: Admitted Workloads - priority: 1 - type: integer - name: v1beta1 - schema: - openAPIV3Schema: - description: ClusterQueue is the Schema for the clusterQueue API. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: ClusterQueueSpec defines the desired state of ClusterQueue - properties: - admissionChecks: - description: |- - admissionChecks lists the AdmissionChecks required by this ClusterQueue. - Cannot be used along with AdmissionCheckStrategy. - items: - type: string - type: array - admissionChecksStrategy: - description: |- - admissionCheckStrategy defines a list of strategies to determine which ResourceFlavors require AdmissionChecks. - This property cannot be used in conjunction with the 'admissionChecks' property. - properties: - admissionChecks: - description: admissionChecks is a list of strategies for AdmissionChecks - items: - description: AdmissionCheckStrategyRule defines rules for a - single AdmissionCheck - properties: - name: - description: name is an AdmissionCheck's name. - type: string - onFlavors: - description: |- - onFlavors is a list of ResourceFlavors' names that this AdmissionCheck should run for. - If empty, the AdmissionCheck will run for all workloads submitted to the ClusterQueue. - items: - description: ResourceFlavorReference is the name of the - ResourceFlavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - type: array - required: - - name - type: object - type: array - type: object - cohort: - description: |- - cohort that this ClusterQueue belongs to. CQs that belong to the - same cohort can borrow unused resources from each other. - - A CQ can be a member of a single borrowing cohort. A workload submitted - to a queue referencing this CQ can borrow quota from any CQ in the cohort. - Only quota for the [resource, flavor] pairs listed in the CQ can be - borrowed. - If empty, this ClusterQueue cannot borrow from any other ClusterQueue and - vice versa. - - A cohort is a name that links CQs together, but it doesn't reference any - object. - - Validation of a cohort name is equivalent to that of object names: - subdomain in DNS (RFC 1123). - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - fairSharing: - description: |- - fairSharing defines the properties of the ClusterQueue when participating in fair sharing. - The values are only relevant if fair sharing is enabled in the Kueue configuration. - properties: - weight: - anyOf: - - type: integer - - type: string - default: 1 - description: |- - weight gives a comparative advantage to this ClusterQueue when competing for unused - resources in the cohort against other ClusterQueues. - The share of a ClusterQueue is based on the dominant resource usage above nominal - quotas for each resource, divided by the weight. - Admission prioritizes scheduling workloads from ClusterQueues with the lowest share - and preempting workloads from the ClusterQueues with the highest share. - A zero weight implies infinite share value, meaning that this ClusterQueue will always - be at disadvantage against other ClusterQueues. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - flavorFungibility: - default: {} - description: |- - flavorFungibility defines whether a workload should try the next flavor - before borrowing or preempting in the flavor being evaluated. - properties: - whenCanBorrow: - default: Borrow - description: |- - whenCanBorrow determines whether a workload should try the next flavor - before borrowing in current flavor. The possible values are: - - - `Borrow` (default): allocate in current flavor if borrowing - is possible. - - `TryNextFlavor`: try next flavor even if the current - flavor has enough resources to borrow. - enum: - - Borrow - - TryNextFlavor - type: string - whenCanPreempt: - default: TryNextFlavor - description: |- - whenCanPreempt determines whether a workload should try the next flavor - before borrowing in current flavor. The possible values are: - - - `Preempt`: allocate in current flavor if it's possible to preempt some workloads. - - `TryNextFlavor` (default): try next flavor even if there are enough - candidates for preemption in the current flavor. - enum: - - Preempt - - TryNextFlavor - type: string - type: object - namespaceSelector: - description: |- - namespaceSelector defines which namespaces are allowed to submit workloads to - this clusterQueue. Beyond this basic support for policy, a policy agent like - Gatekeeper should be used to enforce more advanced policies. - Defaults to null which is a nothing selector (no namespaces eligible). - If set to an empty selector `{}`, then all namespaces are eligible. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - preemption: - default: {} - description: |- - preemption describes policies to preempt Workloads from this ClusterQueue - or the ClusterQueue's cohort. - - Preemption can happen in two scenarios: - - - When a Workload fits within the nominal quota of the ClusterQueue, but - the quota is currently borrowed by other ClusterQueues in the cohort. - Preempting Workloads in other ClusterQueues allows this ClusterQueue to - reclaim its nominal quota. - - When a Workload doesn't fit within the nominal quota of the ClusterQueue - and there are admitted Workloads in the ClusterQueue with lower priority. - - The preemption algorithm tries to find a minimal set of Workloads to - preempt to accomomdate the pending Workload, preempting Workloads with - lower priority first. - properties: - borrowWithinCohort: - default: {} - description: |- - borrowWithinCohort provides configuration to allow preemption within - cohort while borrowing. - properties: - maxPriorityThreshold: - description: |- - maxPriorityThreshold allows to restrict the set of workloads which - might be preempted by a borrowing workload, to only workloads with - priority less than or equal to the specified threshold priority. - When the threshold is not specified, then any workload satisfying the - policy can be preempted by the borrowing workload. - format: int32 - type: integer - policy: - default: Never - description: |- - policy determines the policy for preemption to reclaim quota within cohort while borrowing. - Possible values are: - - `Never` (default): do not allow for preemption, in other - ClusterQueues within the cohort, for a borrowing workload. - - `LowerPriority`: allow preemption, in other ClusterQueues - within the cohort, for a borrowing workload, but only if - the preempted workloads are of lower priority. - enum: - - Never - - LowerPriority - type: string - type: object - reclaimWithinCohort: - default: Never - description: |- - reclaimWithinCohort determines whether a pending Workload can preempt - Workloads from other ClusterQueues in the cohort that are using more than - their nominal quota. The possible values are: - - - `Never` (default): do not preempt Workloads in the cohort. - - `LowerPriority`: **Classic Preemption** if the pending Workload - fits within the nominal quota of its ClusterQueue, only preempt - Workloads in the cohort that have lower priority than the pending - Workload. **Fair Sharing** only preempt Workloads in the cohort that - have lower priority than the pending Workload and that satisfy the - fair sharing preemptionStategies. - - `Any`: **Classic Preemption** if the pending Workload fits within - the nominal quota of its ClusterQueue, preempt any Workload in the - cohort, irrespective of priority. **Fair Sharing** preempt Workloads - in the cohort that satisfy the fair sharing preemptionStrategies. - enum: - - Never - - LowerPriority - - Any - type: string - withinClusterQueue: - default: Never - description: |- - withinClusterQueue determines whether a pending Workload that doesn't fit - within the nominal quota for its ClusterQueue, can preempt active Workloads in - the ClusterQueue. The possible values are: - - - `Never` (default): do not preempt Workloads in the ClusterQueue. - - `LowerPriority`: only preempt Workloads in the ClusterQueue that have - lower priority than the pending Workload. - - `LowerOrNewerEqualPriority`: only preempt Workloads in the ClusterQueue that - either have a lower priority than the pending workload or equal priority - and are newer than the pending workload. - enum: - - Never - - LowerPriority - - LowerOrNewerEqualPriority - type: string - type: object - x-kubernetes-validations: - - message: reclaimWithinCohort=Never and borrowWithinCohort.Policy!=Never - rule: '!(self.reclaimWithinCohort == ''Never'' && has(self.borrowWithinCohort) - && self.borrowWithinCohort.policy != ''Never'')' - queueingStrategy: - default: BestEffortFIFO - description: |- - QueueingStrategy indicates the queueing strategy of the workloads - across the queues in this ClusterQueue. - Current Supported Strategies: - - - StrictFIFO: workloads are ordered strictly by creation time. - Older workloads that can't be admitted will block admitting newer - workloads even if they fit available quota. - - BestEffortFIFO: workloads are ordered by creation time, - however older workloads that can't be admitted will not block - admitting newer workloads that fit existing quota. - enum: - - StrictFIFO - - BestEffortFIFO - type: string - resourceGroups: - description: |- - resourceGroups describes groups of resources. - Each resource group defines the list of resources and a list of flavors - that provide quotas for these resources. - Each resource and each flavor can only form part of one resource group. - resourceGroups can be up to 16. - items: - properties: - coveredResources: - description: |- - coveredResources is the list of resources covered by the flavors in this - group. - Examples: cpu, memory, vendor.com/gpu. - The list cannot be empty and it can contain up to 16 resources. - items: - description: ResourceName is the name identifying various - resources in a ResourceList. - type: string - maxItems: 16 - minItems: 1 - type: array - flavors: - description: |- - flavors is the list of flavors that provide the resources of this group. - Typically, different flavors represent different hardware models - (e.g., gpu models, cpu architectures) or pricing models (on-demand vs spot - cpus). - Each flavor MUST list all the resources listed for this group in the same - order as the .resources field. - The list cannot be empty and it can contain up to 16 flavors. - items: - properties: - name: - description: |- - name of this flavor. The name should match the .metadata.name of a - ResourceFlavor. If a matching ResourceFlavor does not exist, the - ClusterQueue will have an Active condition set to False. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: |- - resources is the list of quotas for this flavor per resource. - There could be up to 16 resources. - items: - properties: - borrowingLimit: - anyOf: - - type: integer - - type: string - description: |- - borrowingLimit is the maximum amount of quota for the [flavor, resource] - combination that this ClusterQueue is allowed to borrow from the unused - quota of other ClusterQueues in the same cohort. - In total, at a given time, Workloads in a ClusterQueue can consume a - quantity of quota equal to nominalQuota+borrowingLimit, assuming the other - ClusterQueues in the cohort have enough unused quota. - If null, it means that there is no borrowing limit. - If not null, it must be non-negative. - borrowingLimit must be null if spec.cohort is empty. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - lendingLimit: - anyOf: - - type: integer - - type: string - description: |- - lendingLimit is the maximum amount of unused quota for the [flavor, resource] - combination that this ClusterQueue can lend to other ClusterQueues in the same cohort. - In total, at a given time, ClusterQueue reserves for its exclusive use - a quantity of quota equals to nominalQuota - lendingLimit. - If null, it means that there is no lending limit, meaning that - all the nominalQuota can be borrowed by other clusterQueues in the cohort. - If not null, it must be non-negative. - lendingLimit must be null if spec.cohort is empty. - This field is in beta stage and is enabled by default. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - name: - description: name of this resource. - type: string - nominalQuota: - anyOf: - - type: integer - - type: string - description: |- - nominalQuota is the quantity of this resource that is available for - Workloads admitted by this ClusterQueue at a point in time. - The nominalQuota must be non-negative. - nominalQuota should represent the resources in the cluster available for - running jobs (after discounting resources consumed by system components - and pods not managed by kueue). In an autoscaled cluster, nominalQuota - should account for resources that can be provided by a component such as - Kubernetes cluster-autoscaler. - - If the ClusterQueue belongs to a cohort, the sum of the quotas for each - (flavor, resource) combination defines the maximum quantity that can be - allocated by a ClusterQueue in the cohort. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - - nominalQuota - type: object - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - coveredResources - - flavors - type: object - x-kubernetes-validations: - - message: flavors must have the same number of resources as the - coveredResources - rule: self.flavors.all(x, size(x.resources) == size(self.coveredResources)) - maxItems: 16 - type: array - x-kubernetes-list-type: atomic - stopPolicy: - default: None - description: |- - stopPolicy - if set to a value different from None, the ClusterQueue is considered Inactive, no new reservation being - made. - - Depending on its value, its associated workloads will: - - - None - Workloads are admitted - - HoldAndDrain - Admitted workloads are evicted and Reserving workloads will cancel the reservation. - - Hold - Admitted workloads will run to completion and Reserving workloads will cancel the reservation. - enum: - - None - - Hold - - HoldAndDrain - type: string - type: object - x-kubernetes-validations: - - message: borrowingLimit must be nil when cohort is empty - rule: '!has(self.cohort) && has(self.resourceGroups) ? self.resourceGroups.all(rg, - rg.flavors.all(f, f.resources.all(r, !has(r.borrowingLimit)))) : true' - status: - description: ClusterQueueStatus defines the observed state of ClusterQueue - properties: - admittedWorkloads: - description: |- - admittedWorkloads is the number of workloads currently admitted to this - clusterQueue and haven't finished yet. - format: int32 - type: integer - conditions: - description: |- - conditions hold the latest available observations of the ClusterQueue - current state. - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - fairSharing: - description: FairSharing contains the information about the current - status of fair sharing. - properties: - weightedShare: - description: |- - WeightedShare represent the maximum of the ratios of usage above nominal - quota to the lendable resources in the cohort, among all the resources - provided by the ClusterQueue, and divided by the weight. - If zero, it means that the usage of the ClusterQueue is below the nominal quota. - If the ClusterQueue has a weight of zero, this will return 9223372036854775807, - the maximum possible share value. - format: int64 - type: integer - required: - - weightedShare - type: object - flavorsReservation: - description: |- - flavorsReservation are the reserved quotas, by flavor, currently in use by the - workloads assigned to this ClusterQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: resources lists the quota usage for the resources - in this flavor. - items: - properties: - borrowed: - anyOf: - - type: integer - - type: string - description: |- - Borrowed is quantity of quota that is borrowed from the cohort. In other - words, it's the used quota that is over the nominalQuota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - name: - description: name of the resource - type: string - total: - anyOf: - - type: integer - - type: string - description: |- - total is the total quantity of used quota, including the amount borrowed - from the cohort. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - flavorsUsage: - description: |- - flavorsUsage are the used quotas, by flavor, currently in use by the - workloads admitted in this ClusterQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: resources lists the quota usage for the resources - in this flavor. - items: - properties: - borrowed: - anyOf: - - type: integer - - type: string - description: |- - Borrowed is quantity of quota that is borrowed from the cohort. In other - words, it's the used quota that is over the nominalQuota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - name: - description: name of the resource - type: string - total: - anyOf: - - type: integer - - type: string - description: |- - total is the total quantity of used quota, including the amount borrowed - from the cohort. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - pendingWorkloads: - description: |- - pendingWorkloads is the number of workloads currently waiting to be - admitted to this clusterQueue. - format: int32 - type: integer - pendingWorkloadsStatus: - description: |- - PendingWorkloadsStatus contains the information exposed about the current - status of the pending workloads in the cluster queue. - Deprecated: This field will be removed on v1beta2, use VisibilityOnDemand - (https://kueue.sigs.k8s.io/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand/) - instead. - properties: - clusterQueuePendingWorkload: - description: Head contains the list of top pending workloads. - items: - description: |- - ClusterQueuePendingWorkload contains the information identifying a pending workload - in the cluster queue. - properties: - name: - description: Name indicates the name of the pending workload. - type: string - namespace: - description: Namespace indicates the name of the pending - workload. - type: string - required: - - name - - namespace - type: object - type: array - x-kubernetes-list-type: atomic - lastChangeTime: - description: LastChangeTime indicates the time of the last change - of the structure. - format: date-time - type: string - required: - - lastChangeTime - type: object - reservingWorkloads: - description: |- - reservingWorkloads is the number of workloads currently reserving quota in this - clusterQueue. - format: int32 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: cohorts.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: Cohort - listKind: CohortList - plural: cohorts - singular: cohort - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: |- - Cohort is the Schema for the cohorts API. Using Hierarchical - Cohorts (any Cohort which has a parent) with Fair Sharing - results in undefined behavior in 0.9 - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: CohortSpec defines the desired state of Cohort - properties: - parent: - description: |- - Parent references the name of the Cohort's parent, if - any. It satisfies one of three cases: - 1) Unset. This Cohort is the root of its Cohort tree. - 2) References a non-existent Cohort. We use default Cohort (no borrowing/lending limits). - 3) References an existent Cohort. - - If a cycle is created, we disable all members of the - Cohort, including ClusterQueues, until the cycle is - removed. We prevent further admission while the cycle - exists. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resourceGroups: - description: |- - ResourceGroups describes groupings of Resources and - Flavors. Each ResourceGroup defines a list of Resources - and a list of Flavors which provide quotas for these - Resources. Each Resource and each Flavor may only form part - of one ResourceGroup. There may be up to 16 ResourceGroups - within a Cohort. - - BorrowingLimit limits how much members of this Cohort - subtree can borrow from the parent subtree. - - LendingLimit limits how much members of this Cohort subtree - can lend to the parent subtree. - - Borrowing and Lending limits must only be set when the - Cohort has a parent. Otherwise, the Cohort create/update - will be rejected by the webhook. - items: - properties: - coveredResources: - description: |- - coveredResources is the list of resources covered by the flavors in this - group. - Examples: cpu, memory, vendor.com/gpu. - The list cannot be empty and it can contain up to 16 resources. - items: - description: ResourceName is the name identifying various - resources in a ResourceList. - type: string - maxItems: 16 - minItems: 1 - type: array - flavors: - description: |- - flavors is the list of flavors that provide the resources of this group. - Typically, different flavors represent different hardware models - (e.g., gpu models, cpu architectures) or pricing models (on-demand vs spot - cpus). - Each flavor MUST list all the resources listed for this group in the same - order as the .resources field. - The list cannot be empty and it can contain up to 16 flavors. - items: - properties: - name: - description: |- - name of this flavor. The name should match the .metadata.name of a - ResourceFlavor. If a matching ResourceFlavor does not exist, the - ClusterQueue will have an Active condition set to False. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: |- - resources is the list of quotas for this flavor per resource. - There could be up to 16 resources. - items: - properties: - borrowingLimit: - anyOf: - - type: integer - - type: string - description: |- - borrowingLimit is the maximum amount of quota for the [flavor, resource] - combination that this ClusterQueue is allowed to borrow from the unused - quota of other ClusterQueues in the same cohort. - In total, at a given time, Workloads in a ClusterQueue can consume a - quantity of quota equal to nominalQuota+borrowingLimit, assuming the other - ClusterQueues in the cohort have enough unused quota. - If null, it means that there is no borrowing limit. - If not null, it must be non-negative. - borrowingLimit must be null if spec.cohort is empty. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - lendingLimit: - anyOf: - - type: integer - - type: string - description: |- - lendingLimit is the maximum amount of unused quota for the [flavor, resource] - combination that this ClusterQueue can lend to other ClusterQueues in the same cohort. - In total, at a given time, ClusterQueue reserves for its exclusive use - a quantity of quota equals to nominalQuota - lendingLimit. - If null, it means that there is no lending limit, meaning that - all the nominalQuota can be borrowed by other clusterQueues in the cohort. - If not null, it must be non-negative. - lendingLimit must be null if spec.cohort is empty. - This field is in beta stage and is enabled by default. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - name: - description: name of this resource. - type: string - nominalQuota: - anyOf: - - type: integer - - type: string - description: |- - nominalQuota is the quantity of this resource that is available for - Workloads admitted by this ClusterQueue at a point in time. - The nominalQuota must be non-negative. - nominalQuota should represent the resources in the cluster available for - running jobs (after discounting resources consumed by system components - and pods not managed by kueue). In an autoscaled cluster, nominalQuota - should account for resources that can be provided by a component such as - Kubernetes cluster-autoscaler. - - If the ClusterQueue belongs to a cohort, the sum of the quotas for each - (flavor, resource) combination defines the maximum quantity that can be - allocated by a ClusterQueue in the cohort. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - - nominalQuota - type: object - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - coveredResources - - flavors - type: object - x-kubernetes-validations: - - message: flavors must have the same number of resources as the - coveredResources - rule: self.flavors.all(x, size(x.resources) == size(self.coveredResources)) - maxItems: 16 - type: array - x-kubernetes-list-type: atomic - type: object - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: localqueues.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: LocalQueue - listKind: LocalQueueList - plural: localqueues - shortNames: - - queue - - queues - - lq - singular: localqueue - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Backing ClusterQueue - jsonPath: .spec.clusterQueue - name: ClusterQueue - type: string - - description: Number of pending workloads - jsonPath: .status.pendingWorkloads - name: Pending Workloads - type: integer - - description: Number of admitted workloads that haven't finished yet. - jsonPath: .status.admittedWorkloads - name: Admitted Workloads - type: integer - name: v1beta1 - schema: - openAPIV3Schema: - description: LocalQueue is the Schema for the localQueues API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: LocalQueueSpec defines the desired state of LocalQueue - properties: - clusterQueue: - description: clusterQueue is a reference to a clusterQueue that backs - this localQueue. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - x-kubernetes-validations: - - message: field is immutable - rule: self == oldSelf - stopPolicy: - default: None - description: |- - stopPolicy - if set to a value different from None, the LocalQueue is considered Inactive, - no new reservation being made. - - Depending on its value, its associated workloads will: - - - None - Workloads are admitted - - HoldAndDrain - Admitted workloads are evicted and Reserving workloads will cancel the reservation. - - Hold - Admitted workloads will run to completion and Reserving workloads will cancel the reservation. - enum: - - None - - Hold - - HoldAndDrain - type: string - type: object - status: - description: LocalQueueStatus defines the observed state of LocalQueue - properties: - admittedWorkloads: - description: |- - admittedWorkloads is the number of workloads in this LocalQueue - admitted to a ClusterQueue and that haven't finished yet. - format: int32 - type: integer - conditions: - description: |- - Conditions hold the latest available observations of the LocalQueue - current state. - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - flavorUsage: - description: |- - flavorsUsage are the used quotas, by flavor currently in use by the - workloads assigned to this LocalQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: resources lists the quota usage for the resources - in this flavor. - items: - properties: - name: - description: name of the resource. - type: string - total: - anyOf: - - type: integer - - type: string - description: total is the total quantity of used quota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - flavors: - description: flavors lists all currently available ResourceFlavors - in specified ClusterQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - nodeLabels: - additionalProperties: - type: string - description: |- - nodeLabels are labels that associate the ResourceFlavor with Nodes that - have the same labels. - maxProperties: 8 - type: object - x-kubernetes-map-type: atomic - nodeTaints: - description: |- - nodeTaints are taints that the nodes associated with this ResourceFlavor - have. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Required. The taint key to be applied to - a node. - type: string - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint - key. - type: string - required: - - effect - - key - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - resources: - description: resources used in the flavor. - items: - description: ResourceName is the name identifying various - resources in a ResourceList. - type: string - maxItems: 16 - type: array - x-kubernetes-list-type: set - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - flavorsReservation: - description: |- - flavorsReservation are the reserved quotas, by flavor currently in use by the - workloads assigned to this LocalQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: resources lists the quota usage for the resources - in this flavor. - items: - properties: - name: - description: name of the resource. - type: string - total: - anyOf: - - type: integer - - type: string - description: total is the total quantity of used quota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - pendingWorkloads: - description: PendingWorkloads is the number of Workloads in the LocalQueue - not yet admitted to a ClusterQueue - format: int32 - type: integer - reservingWorkloads: - description: |- - reservingWorkloads is the number of workloads in this LocalQueue - reserving quota in a ClusterQueue and that haven't finished yet. - format: int32 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: multikueueclusters.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: MultiKueueCluster - listKind: MultiKueueClusterList - plural: multikueueclusters - singular: multikueuecluster - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: MultiKueueCluster is the Schema for the multikueue API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - properties: - kubeConfig: - description: Information how to connect to the cluster. - properties: - location: - description: |- - Location of the KubeConfig. - - If LocationType is Secret then Location is the name of the secret inside the namespace in - which the kueue controller manager is running. The config should be stored in the "kubeconfig" key. - type: string - locationType: - default: Secret - description: Type of the KubeConfig location. - enum: - - Secret - - Path - type: string - required: - - location - - locationType - type: object - required: - - kubeConfig - type: object - status: - properties: - conditions: - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: multikueueconfigs.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: MultiKueueConfig - listKind: MultiKueueConfigList - plural: multikueueconfigs - singular: multikueueconfig - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: MultiKueueConfig is the Schema for the multikueue API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: MultiKueueConfigSpec defines the desired state of MultiKueueConfig - properties: - clusters: - description: List of MultiKueueClusters names where the workloads - from the ClusterQueue should be distributed. - items: - type: string - maxItems: 10 - minItems: 1 - type: array - x-kubernetes-list-type: set - required: - - clusters - type: object - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: provisioningrequestconfigs.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: ProvisioningRequestConfig - listKind: ProvisioningRequestConfigList - plural: provisioningrequestconfigs - singular: provisioningrequestconfig - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: ProvisioningRequestConfig is the Schema for the provisioningrequestconfig - API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: ProvisioningRequestConfigSpec defines the desired state of - ProvisioningRequestConfig - properties: - managedResources: - description: |- - managedResources contains the list of resources managed by the autoscaling. - - If empty, all resources are considered managed. - - If not empty, the ProvisioningRequest will contain only the podsets that are - requesting at least one of them. - - If none of the workloads podsets is requesting at least a managed resource, - the workload is considered ready. - items: - description: ResourceName is the name identifying various resources - in a ResourceList. - type: string - maxItems: 100 - type: array - x-kubernetes-list-type: set - parameters: - additionalProperties: - description: Parameter is limited to 255 characters. - maxLength: 255 - type: string - description: Parameters contains all other parameters classes may - require. - maxProperties: 100 - type: object - provisioningClassName: - description: |- - ProvisioningClassName describes the different modes of provisioning the resources. - Check autoscaling.x-k8s.io ProvisioningRequestSpec.ProvisioningClassName for details. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - retryStrategy: - default: - backoffBaseSeconds: 60 - backoffLimitCount: 3 - backoffMaxSeconds: 1800 - description: |- - retryStrategy defines strategy for retrying ProvisioningRequest. - If null, then the default configuration is applied with the following parameter values: - backoffLimitCount: 3 - backoffBaseSeconds: 60 - 1 min - backoffMaxSeconds: 1800 - 30 mins - - To switch off retry mechanism - set retryStrategy.backoffLimitCount to 0. - properties: - backoffBaseSeconds: - default: 60 - description: |- - BackoffBaseSeconds defines the base for the exponential backoff for - re-queuing an evicted workload. - - Defaults to 60. - format: int32 - type: integer - backoffLimitCount: - default: 3 - description: |- - BackoffLimitCount defines the maximum number of re-queuing retries. - Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). - - Every backoff duration is about "b*2^(n-1)+Rand" where: - - "b" represents the base set by "BackoffBaseSeconds" parameter, - - "n" represents the "workloadStatus.requeueState.count", - - "Rand" represents the random jitter. - During this time, the workload is taken as an inadmissible and - other workloads will have a chance to be admitted. - By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...). - - Defaults to 3. - format: int32 - type: integer - backoffMaxSeconds: - default: 1800 - description: |- - BackoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload. - - Defaults to 1800. - format: int32 - type: integer - type: object - required: - - provisioningClassName - type: object - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: resourceflavors.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: ResourceFlavor - listKind: ResourceFlavorList - plural: resourceflavors - shortNames: - - flavor - - flavors - - rf - singular: resourceflavor - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: ResourceFlavor is the Schema for the resourceflavors API. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: ResourceFlavorSpec defines the desired state of the ResourceFlavor - properties: - nodeLabels: - additionalProperties: - type: string - description: |- - nodeLabels are labels that associate the ResourceFlavor with Nodes that - have the same labels. - When a Workload is admitted, its podsets can only get assigned - ResourceFlavors whose nodeLabels match the nodeSelector and nodeAffinity - fields. - Once a ResourceFlavor is assigned to a podSet, the ResourceFlavor's - nodeLabels should be injected into the pods of the Workload by the - controller that integrates with the Workload object. - - nodeLabels can be up to 8 elements. - maxProperties: 8 - type: object - x-kubernetes-map-type: atomic - nodeTaints: - description: |- - nodeTaints are taints that the nodes associated with this ResourceFlavor - have. - Workloads' podsets must have tolerations for these nodeTaints in order to - get assigned this ResourceFlavor during admission. - - An example of a nodeTaint is - cloud.provider.com/preemptible="true":NoSchedule - - nodeTaints can be up to 8 elements. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Required. The taint key to be applied to a node. - type: string - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - required: - - effect - - key - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - x-kubernetes-validations: - - message: 'supported taint effect values: ''NoSchedule'', ''PreferNoSchedule'', - ''NoExecute''' - rule: self.all(x, x.effect in ['NoSchedule', 'PreferNoSchedule', - 'NoExecute']) - tolerations: - description: |- - tolerations are extra tolerations that will be added to the pods admitted in - the quota associated with this resource flavor. - - An example of a toleration is - cloud.provider.com/preemptible="true":NoSchedule - - tolerations can be up to 8 elements. - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - x-kubernetes-validations: - - message: operator must be Exists when 'key' is empty, which means - 'match all values and all keys' - rule: 'self.all(x, !has(x.key) ? x.operator == ''Exists'' : true)' - - message: effect must be 'NoExecute' when 'tolerationSeconds' is - set - rule: 'self.all(x, has(x.tolerationSeconds) ? x.effect == ''NoExecute'' - : true)' - - message: 'supported toleration values: ''Equal''(default), ''Exists''' - rule: self.all(x, !has(x.operator) || x.operator in ['Equal', 'Exists']) - - message: a value must be empty when 'operator' is 'Exists' - rule: 'self.all(x, has(x.operator) && x.operator == ''Exists'' ? - !has(x.value) : true)' - - message: 'supported taint effect values: ''NoSchedule'', ''PreferNoSchedule'', - ''NoExecute''' - rule: self.all(x, !has(x.effect) || x.effect in ['NoSchedule', 'PreferNoSchedule', - 'NoExecute']) - topologyName: - description: |- - topologyName indicates topology for the TAS ResourceFlavor. - When specified, it enables scraping of the topology information from the - nodes matching to the Resource Flavor node labels. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - type: object - x-kubernetes-validations: - - message: at least one nodeLabel is required when topology is set - rule: '!has(self.topologyName) || self.nodeLabels.size() >= 1' - - message: resourceFlavorSpec are immutable when topologyName is set - rule: '!has(oldSelf.topologyName) || self == oldSelf' - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: topologies.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: Topology - listKind: TopologyList - plural: topologies - singular: topology - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: Topology is the Schema for the topology API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: TopologySpec defines the desired state of Topology - properties: - levels: - description: levels define the levels of topology. - items: - description: TopologyLevel defines the desired state of TopologyLevel - properties: - nodeLabel: - description: |- - nodeLabel indicates the name of the node label for a specific topology - level. - - Examples: - - cloud.provider.com/topology-block - - cloud.provider.com/topology-rack - maxLength: 316 - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - nodeLabel - type: object - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - x-kubernetes-validations: - - message: field is immutable - rule: self == oldSelf - - message: must be unique - rule: size(self.filter(i, size(self.filter(j, j == i)) > 1)) == - 0 - - message: the kubernetes.io/hostname label can only be used at the - lowest level of topology - rule: size(self.filter(i, i.nodeLabel == 'kubernetes.io/hostname')) - == 0 || self[size(self) - 1].nodeLabel == 'kubernetes.io/hostname' - required: - - levels - type: object - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: workloadpriorityclasses.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: WorkloadPriorityClass - listKind: WorkloadPriorityClassList - plural: workloadpriorityclasses - singular: workloadpriorityclass - scope: Cluster - versions: - - additionalPrinterColumns: - - description: Value of workloadPriorityClass's Priority - jsonPath: .value - name: Value - type: integer - name: v1beta1 - schema: - openAPIV3Schema: - description: WorkloadPriorityClass is the Schema for the workloadPriorityClass - API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - description: - description: |- - description is an arbitrary string that usually provides guidelines on - when this workloadPriorityClass should be used. - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - value: - description: |- - value represents the integer value of this workloadPriorityClass. This is the actual priority that workloads - receive when jobs have the name of this class in their workloadPriorityClass label. - Changing the value of workloadPriorityClass doesn't affect the priority of workloads that were already created. - format: int32 - type: integer - required: - - value - type: object - served: true - storage: true - subresources: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: workloads.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: Workload - listKind: WorkloadList - plural: workloads - shortNames: - - wl - singular: workload - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Name of the queue this workload was submitted to - jsonPath: .spec.queueName - name: Queue - type: string - - description: Name of the ClusterQueue where the workload is reserving quota - jsonPath: .status.admission.clusterQueue - name: Reserved in - type: string - - description: Admission status - jsonPath: .status.conditions[?(@.type=='Admitted')].status - name: Admitted - type: string - - description: Workload finished - jsonPath: .status.conditions[?(@.type=='Finished')].status - name: Finished - type: string - - description: Time this workload was created - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: Workload is the Schema for the workloads API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: WorkloadSpec defines the desired state of Workload - properties: - active: - default: true - description: |- - Active determines if a workload can be admitted into a queue. - Changing active from true to false will evict any running workloads. - Possible values are: - - - false: indicates that a workload should never be admitted and evicts running workloads - - true: indicates that a workload can be evaluated for admission into it's respective queue. - - Defaults to true - type: boolean - maximumExecutionTimeSeconds: - description: |- - maximumExecutionTimeSeconds if provided, determines the maximum time, in seconds, - the workload can be admitted before it's automatically deactivated. - - If unspecified, no execution time limit is enforced on the Workload. - format: int32 - minimum: 1 - type: integer - podSets: - description: |- - podSets is a list of sets of homogeneous pods, each described by a Pod spec - and a count. - There must be at least one element and at most 8. - podSets cannot be changed. - items: - properties: - count: - default: 1 - description: count is the number of pods for the spec. - format: int32 - minimum: 0 - type: integer - minCount: - description: |- - minCount is the minimum number of pods for the spec acceptable - if the workload supports partial admission. - - If not provided, partial admission for the current PodSet is not - enabled. - - Only one podSet within the workload can use this. - - This is an alpha field and requires enabling PartialAdmission feature gate. - format: int32 - minimum: 1 - type: integer - name: - default: main - description: name is the PodSet name. - maxLength: 63 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - template: - description: |- - template is the Pod template. - - The only allowed fields in template.metadata are labels and annotations. - - If requests are omitted for a container or initContainer, - they default to the limits if they are explicitly specified for the - container or initContainer. - - During admission, the rules in nodeSelector and - nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution that match - the keys in the nodeLabels from the ResourceFlavors considered for this - Workload are used to filter the ResourceFlavors that can be assigned to - this podSet. - properties: - metadata: - description: |- - Standard object's metadata. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - Specification of the desired behavior of the pod. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - activeDeadlineSeconds: - description: |- - Optional duration in seconds the pod may be active on the node relative to - StartTime before the system will actively try to mark it failed and kill associated containers. - Value must be a positive integer. - format: int64 - type: integer - affinity: - description: If specified, the pod's scheduling constraints - properties: - nodeAffinity: - description: Describes node affinity scheduling - rules for the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node matches the corresponding matchExpressions; the - node(s) with the highest sum are the most preferred. - items: - description: |- - An empty preferred scheduling term matches all objects with implicit weight 0 - (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated - with the corresponding weight. - properties: - matchExpressions: - description: A list of node selector - requirements by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector - requirements by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - description: Weight associated with matching - the corresponding nodeSelectorTerm, - in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector - terms. The terms are ORed. - items: - description: |- - A null or empty node selector term matches no objects. The requirements of - them are ANDed. - The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector - requirements by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector - requirements by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - description: Describes pod affinity scheduling rules - (e.g. co-locate this pod in the same node, zone, - etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched - WeightedPodAffinityTerm fields are added - per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity - term, associated with the corresponding - weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a - list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a - list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling - rules (e.g. avoid putting this pod in the same - node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the anti-affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched - WeightedPodAffinityTerm fields are added - per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity - term, associated with the corresponding - weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the anti-affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the anti-affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a - list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a - list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - automountServiceAccountToken: - description: AutomountServiceAccountToken indicates - whether a service account token should be automatically - mounted. - type: boolean - containers: - description: |- - List of containers belonging to the pod. - Containers cannot currently be added or removed. - There must be at least one container in a Pod. - Cannot be updated. - items: - description: A single application container that you - want to run within a pod. - properties: - args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment - variable present in a Container. - properties: - name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment - variable's value. Cannot be used if value - is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret - in the pod's namespace - properties: - key: - description: The key of the secret - to select from. Must be a valid - secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source - of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend - to each key in the ConfigMap. Must be - a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action - to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration - that the container should sleep before - being terminated. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action - to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration - that the container should sleep before - being terminated. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action - involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. - type: string - ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network - port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external - port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action - involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents - resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. - type: string - securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label - that applies to the container. - type: string - role: - description: Role is a SELinux role label - that applies to the container. - type: string - type: - description: Type is a SELinux type label - that applies to the container. - type: string - user: - description: User is a SELinux user label - that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is - the name of the GMSA credential spec - to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action - involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block - devices to be used by the container. - items: - description: volumeDevice describes a mapping - of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside - of the container that the device will - be mapped to. - type: string - name: - description: name must match the name of - a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. - items: - description: VolumeMount describes a mounting - of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of - a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - dnsConfig: - description: |- - Specifies the DNS parameters of a pod. - Parameters specified here will be merged to the generated DNS - configuration based on DNSPolicy. - properties: - nameservers: - description: |- - A list of DNS name server IP addresses. - This will be appended to the base nameservers generated from DNSPolicy. - Duplicated nameservers will be removed. - items: - type: string - type: array - x-kubernetes-list-type: atomic - options: - description: |- - A list of DNS resolver options. - This will be merged with the base options generated from DNSPolicy. - Duplicated entries will be removed. Resolution options given in Options - will override those that appear in the base DNSPolicy. - items: - description: PodDNSConfigOption defines DNS resolver - options of a pod. - properties: - name: - description: Required. - type: string - value: - type: string - type: object - type: array - x-kubernetes-list-type: atomic - searches: - description: |- - A list of DNS search domains for host-name lookup. - This will be appended to the base search paths generated from DNSPolicy. - Duplicated search paths will be removed. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - dnsPolicy: - description: |- - Set DNS policy for the pod. - Defaults to "ClusterFirst". - Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. - DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. - To have DNS options set along with hostNetwork, you have to specify DNS policy - explicitly to 'ClusterFirstWithHostNet'. - type: string - enableServiceLinks: - description: |- - EnableServiceLinks indicates whether information about services should be injected into pod's - environment variables, matching the syntax of Docker links. - Optional: Defaults to true. - type: boolean - ephemeralContainers: - description: |- - List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing - pod to perform user-initiated actions such as debugging. This list cannot be specified when - creating a pod, and it cannot be modified by updating the pod spec. In order to add an - ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. - items: - description: |- - An EphemeralContainer is a temporary container that you may add to an existing Pod for - user-initiated activities such as debugging. Ephemeral containers have no resource or - scheduling guarantees, and they will not be restarted when they exit or when a Pod is - removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the - Pod to exceed its resource allocation. - - To add an ephemeral container, use the ephemeralcontainers subresource of an existing - Pod. Ephemeral containers may not be removed or restarted. - properties: - args: - description: |- - Arguments to the entrypoint. - The image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment - variable present in a Container. - properties: - name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment - variable's value. Cannot be used if value - is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret - in the pod's namespace - properties: - key: - description: The key of the secret - to select from. Must be a valid - secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source - of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend - to each key in the ConfigMap. Must be - a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: Lifecycle is not allowed for ephemeral - containers. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action - to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration - that the container should sleep before - being terminated. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action - to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration - that the container should sleep before - being terminated. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: Probes are not allowed for ephemeral - containers. - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action - involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the ephemeral container specified as a DNS_LABEL. - This name must be unique among all containers, init containers and ephemeral containers. - type: string - ports: - description: Ports are not allowed for ephemeral - containers. - items: - description: ContainerPort represents a network - port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external - port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: Probes are not allowed for ephemeral - containers. - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action - involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents - resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources - already allocated to the pod. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - Restart policy for the container to manage the restart behavior of each - container within a pod. - This may only be set for init containers. You cannot set this field on - ephemeral containers. - type: string - securityContext: - description: |- - Optional: SecurityContext defines the security options the ephemeral container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label - that applies to the container. - type: string - role: - description: Role is a SELinux role label - that applies to the container. - type: string - type: - description: Type is a SELinux type label - that applies to the container. - type: string - user: - description: User is a SELinux user label - that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is - the name of the GMSA credential spec - to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: Probes are not allowed for ephemeral - containers. - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action - involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - targetContainerName: - description: |- - If set, the name of the container from PodSpec that this ephemeral container targets. - The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. - If not set then the ephemeral container uses the namespaces configured in the Pod spec. - - The container runtime must implement support for this feature. If the runtime does not - support namespace targeting then the result of setting this field is undefined. - type: string - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block - devices to be used by the container. - items: - description: volumeDevice describes a mapping - of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside - of the container that the device will - be mapped to. - type: string - name: - description: name must match the name of - a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. - Cannot be updated. - items: - description: VolumeMount describes a mounting - of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of - a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - hostAliases: - description: |- - HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts - file if specified. - items: - description: |- - HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the - pod's hosts file. - properties: - hostnames: - description: Hostnames for the above IP address. - items: - type: string - type: array - x-kubernetes-list-type: atomic - ip: - description: IP address of the host file entry. - type: string - required: - - ip - type: object - type: array - x-kubernetes-list-map-keys: - - ip - x-kubernetes-list-type: map - hostIPC: - description: |- - Use the host's ipc namespace. - Optional: Default to false. - type: boolean - hostNetwork: - description: |- - Host networking requested for this pod. Use the host's network namespace. - If this option is set, the ports that will be used must be specified. - Default to false. - type: boolean - hostPID: - description: |- - Use the host's pid namespace. - Optional: Default to false. - type: boolean - hostUsers: - description: |- - Use the host's user namespace. - Optional: Default to true. - If set to true or not present, the pod will be run in the host user namespace, useful - for when the pod needs a feature only available to the host user namespace, such as - loading a kernel module with CAP_SYS_MODULE. - When set to false, a new userns is created for the pod. Setting false is useful for - mitigating container breakout vulnerabilities even allowing users to run their - containers as root without actually having root privileges on the host. - This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. - type: boolean - hostname: - description: |- - Specifies the hostname of the Pod - If not specified, the pod's hostname will be set to a system-defined value. - type: string - imagePullSecrets: - description: |- - ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. - If specified, these secrets will be passed to individual puller implementations for them to use. - More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod - items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - initContainers: - description: |- - List of initialization containers belonging to the pod. - Init containers are executed in order prior to containers being started. If any - init container fails, the pod is considered to have failed and is handled according - to its restartPolicy. The name for an init container or normal container must be - unique among all containers. - Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. - The resourceRequirements of an init container are taken into account during scheduling - by finding the highest request/limit for each resource type, and then using the max of - of that value or the sum of the normal containers. Limits are applied to init containers - in a similar fashion. - Init containers cannot currently be added or removed. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ - items: - description: A single application container that you - want to run within a pod. - properties: - args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment - variable present in a Container. - properties: - name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment - variable's value. Cannot be used if value - is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret - in the pod's namespace - properties: - key: - description: The key of the secret - to select from. Must be a valid - secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source - of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend - to each key in the ConfigMap. Must be - a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action - to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration - that the container should sleep before - being terminated. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action - to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration - that the container should sleep before - being terminated. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action - involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. - type: string - ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network - port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external - port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action - involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents - resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. - type: string - securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label - that applies to the container. - type: string - role: - description: Role is a SELinux role label - that applies to the container. - type: string - type: - description: Type is a SELinux type label - that applies to the container. - type: string - user: - description: User is a SELinux user label - that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is - the name of the GMSA credential spec - to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action - involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block - devices to be used by the container. - items: - description: volumeDevice describes a mapping - of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside - of the container that the device will - be mapped to. - type: string - name: - description: name must match the name of - a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. - items: - description: VolumeMount describes a mounting - of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of - a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - nodeName: - description: |- - NodeName indicates in which node this pod is scheduled. - If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. - Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. - This field should not be used to express a desire for the pod to be scheduled on a specific node. - https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename - type: string - nodeSelector: - additionalProperties: - type: string - description: |- - NodeSelector is a selector which must be true for the pod to fit on a node. - Selector which must match a node's labels for the pod to be scheduled on that node. - More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - type: object - x-kubernetes-map-type: atomic - os: - description: |- - Specifies the OS of the containers in the pod. - Some pod and container fields are restricted if this is set. - - If the OS field is set to linux, the following fields must be unset: - -securityContext.windowsOptions - - If the OS field is set to windows, following fields must be unset: - - spec.hostPID - - spec.hostIPC - - spec.hostUsers - - spec.securityContext.appArmorProfile - - spec.securityContext.seLinuxOptions - - spec.securityContext.seccompProfile - - spec.securityContext.fsGroup - - spec.securityContext.fsGroupChangePolicy - - spec.securityContext.sysctls - - spec.shareProcessNamespace - - spec.securityContext.runAsUser - - spec.securityContext.runAsGroup - - spec.securityContext.supplementalGroups - - spec.securityContext.supplementalGroupsPolicy - - spec.containers[*].securityContext.appArmorProfile - - spec.containers[*].securityContext.seLinuxOptions - - spec.containers[*].securityContext.seccompProfile - - spec.containers[*].securityContext.capabilities - - spec.containers[*].securityContext.readOnlyRootFilesystem - - spec.containers[*].securityContext.privileged - - spec.containers[*].securityContext.allowPrivilegeEscalation - - spec.containers[*].securityContext.procMount - - spec.containers[*].securityContext.runAsUser - - spec.containers[*].securityContext.runAsGroup - properties: - name: - description: |- - Name is the name of the operating system. The currently supported values are linux and windows. - Additional value may be defined in future and can be one of: - https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration - Clients should expect to handle additional values and treat unrecognized values in this field as os: null - type: string - required: - - name - type: object - overhead: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. - This field will be autopopulated at admission time by the RuntimeClass admission controller. If - the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. - The RuntimeClass admission controller will reject Pod create requests which have the overhead already - set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value - defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. - More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md - type: object - preemptionPolicy: - description: |- - PreemptionPolicy is the Policy for preempting pods with lower priority. - One of Never, PreemptLowerPriority. - Defaults to PreemptLowerPriority if unset. - type: string - priority: - description: |- - The priority value. Various system components use this field to find the - priority of the pod. When Priority Admission Controller is enabled, it - prevents users from setting this field. The admission controller populates - this field from PriorityClassName. - The higher the value, the higher the priority. - format: int32 - type: integer - priorityClassName: - description: |- - If specified, indicates the pod's priority. "system-node-critical" and - "system-cluster-critical" are two special keywords which indicate the - highest priorities with the former being the highest priority. Any other - name must be defined by creating a PriorityClass object with that name. - If not specified, the pod priority will be default or zero if there is no - default. - type: string - readinessGates: - description: |- - If specified, all readiness gates will be evaluated for pod readiness. - A pod is ready when all its containers are ready AND - all conditions specified in the readiness gates have status equal to "True" - More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates - items: - description: PodReadinessGate contains the reference - to a pod condition - properties: - conditionType: - description: ConditionType refers to a condition - in the pod's condition list with matching type. - type: string - required: - - conditionType - type: object - type: array - x-kubernetes-list-type: atomic - resourceClaims: - description: |- - ResourceClaims defines which ResourceClaims must be allocated - and reserved before the Pod is allowed to start. The resources - will be made available to those containers which consume them - by name. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. - items: - description: |- - PodResourceClaim references exactly one ResourceClaim, either directly - or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim - for the pod. - - It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. - Containers that need access to the ResourceClaim reference it with this name. - properties: - name: - description: |- - Name uniquely identifies this resource claim inside the pod. - This must be a DNS_LABEL. - type: string - resourceClaimName: - description: |- - ResourceClaimName is the name of a ResourceClaim object in the same - namespace as this pod. - - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. - type: string - resourceClaimTemplateName: - description: |- - ResourceClaimTemplateName is the name of a ResourceClaimTemplate - object in the same namespace as this pod. - - The template will be used to create a new ResourceClaim, which will - be bound to this pod. When this pod is deleted, the ResourceClaim - will also be deleted. The pod name and resource name, along with a - generated component, will be used to form a unique name for the - ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. - - This field is immutable and no changes will be made to the - corresponding ResourceClaim by the control plane after creating the - ResourceClaim. - - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - restartPolicy: - description: |- - Restart policy for all containers within the pod. - One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. - Default to Always. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy - type: string - runtimeClassName: - description: |- - RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used - to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. - If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an - empty definition that uses the default runtime handler. - More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class - type: string - schedulerName: - description: |- - If specified, the pod will be dispatched by specified scheduler. - If not specified, the pod will be dispatched by default scheduler. - type: string - schedulingGates: - description: |- - SchedulingGates is an opaque list of values that if specified will block scheduling the pod. - If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the - scheduler will not attempt to schedule the pod. - - SchedulingGates can only be set at pod creation time, and be removed only afterwards. - items: - description: PodSchedulingGate is associated to a - Pod to guard its scheduling. - properties: - name: - description: |- - Name of the scheduling gate. - Each scheduling gate must have a unique name field. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - securityContext: - description: |- - SecurityContext holds pod-level security attributes and common container settings. - Optional: Defaults to empty. See type description for default values of each field. - properties: - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - fsGroup: - description: |- - A special supplemental group that applies to all containers in a pod. - Some volume types allow the Kubelet to change the ownership of that volume - to be owned by the pod: - - 1. The owning GID will be the FSGroup - 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) - 3. The permission bits are OR'd with rw-rw---- - - If unset, the Kubelet will not modify the ownership and permissions of any volume. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - fsGroupChangePolicy: - description: |- - fsGroupChangePolicy defines behavior of changing ownership and permission of the volume - before being exposed inside Pod. This field will only apply to - volume types which support fsGroup based ownership(and permissions). - It will have no effect on ephemeral volume types such as: secret, configmaps - and emptydir. - Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. - Note that this field cannot be set when spec.os.name is windows. - type: string - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in SecurityContext. If set in - both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that - applies to the container. - type: string - role: - description: Role is a SELinux role label that - applies to the container. - type: string - type: - description: Type is a SELinux type label that - applies to the container. - type: string - user: - description: User is a SELinux user label that - applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - supplementalGroups: - description: |- - A list of groups applied to the first process run in each container, in - addition to the container's primary GID and fsGroup (if specified). If - the SupplementalGroupsPolicy feature is enabled, the - supplementalGroupsPolicy field determines whether these are in addition - to or instead of any group memberships defined in the container image. - If unspecified, no additional groups are added, though group memberships - defined in the container image may still be used, depending on the - supplementalGroupsPolicy field. - Note that this field cannot be set when spec.os.name is windows. - items: - format: int64 - type: integer - type: array - x-kubernetes-list-type: atomic - supplementalGroupsPolicy: - description: |- - Defines how supplemental groups of the first container processes are calculated. - Valid values are "Merge" and "Strict". If not specified, "Merge" is used. - (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled - and the container runtime must implement support for this feature. - Note that this field cannot be set when spec.os.name is windows. - type: string - sysctls: - description: |- - Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported - sysctls (by the container runtime) might fail to launch. - Note that this field cannot be set when spec.os.name is windows. - items: - description: Sysctl defines a kernel parameter - to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options within a container's SecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name - of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - serviceAccount: - description: |- - DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. - Deprecated: Use serviceAccountName instead. - type: string - serviceAccountName: - description: |- - ServiceAccountName is the name of the ServiceAccount to use to run this pod. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ - type: string - setHostnameAsFQDN: - description: |- - If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). - In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). - In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. - If a pod does not have FQDN, this has no effect. - Default to false. - type: boolean - shareProcessNamespace: - description: |- - Share a single process namespace between all of the containers in a pod. - When this is set containers will be able to view and signal processes from other containers - in the same pod, and the first process in each container will not be assigned PID 1. - HostPID and ShareProcessNamespace cannot both be set. - Optional: Default to false. - type: boolean - subdomain: - description: |- - If specified, the fully qualified Pod hostname will be "...svc.". - If not specified, the pod will not have a domainname at all. - type: string - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - If this value is nil, the default grace period will be used instead. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - Defaults to 30 seconds. - format: int64 - type: integer - tolerations: - description: If specified, the pod's tolerations. - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - topologySpreadConstraints: - description: |- - TopologySpreadConstraints describes how a group of pods ought to spread across topology - domains. Scheduler will schedule pods in a way which abides by the constraints. - All topologySpreadConstraints are ANDed. - items: - description: TopologySpreadConstraint specifies how - to spread matching pods among the given topology. - properties: - labelSelector: - description: |- - LabelSelector is used to find matching pods. - Pods that match this label selector are counted to determine the number of pods - in their corresponding topology domain. - properties: - matchExpressions: - description: matchExpressions is a list of - label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select the pods over which - spreading will be calculated. The keys are used to lookup values from the - incoming pod labels, those key-value labels are ANDed with labelSelector - to select the group of existing pods over which spreading will be calculated - for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - MatchLabelKeys cannot be set when LabelSelector isn't set. - Keys that don't exist in the incoming pod labels will - be ignored. A null or empty list means only match against labelSelector. - - This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxSkew: - description: |- - MaxSkew describes the degree to which pods may be unevenly distributed. - When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference - between the number of matching pods in the target topology and the global minimum. - The global minimum is the minimum number of matching pods in an eligible domain - or zero if the number of eligible domains is less than MinDomains. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 2/2/1: - In this case, the global minimum is 1. - | zone1 | zone2 | zone3 | - | P P | P P | P | - - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; - scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) - violate MaxSkew(1). - - if MaxSkew is 2, incoming pod can be scheduled onto any zone. - When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence - to topologies that satisfy it. - It's a required field. Default value is 1 and 0 is not allowed. - format: int32 - type: integer - minDomains: - description: |- - MinDomains indicates a minimum number of eligible domains. - When the number of eligible domains with matching topology keys is less than minDomains, - Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. - And when the number of eligible domains with matching topology keys equals or greater than minDomains, - this value has no effect on scheduling. - As a result, when the number of eligible domains is less than minDomains, - scheduler won't schedule more than maxSkew Pods to those domains. - If value is nil, the constraint behaves as if MinDomains is equal to 1. - Valid values are integers greater than 0. - When value is not nil, WhenUnsatisfiable must be DoNotSchedule. - - For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same - labelSelector spread as 2/2/2: - | zone1 | zone2 | zone3 | - | P P | P P | P P | - The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. - In this situation, new pod with the same labelSelector cannot be scheduled, - because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, - it will violate MaxSkew. - format: int32 - type: integer - nodeAffinityPolicy: - description: |- - NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector - when calculating pod topology spread skew. Options are: - - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. - - If this value is nil, the behavior is equivalent to the Honor policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. - type: string - nodeTaintsPolicy: - description: |- - NodeTaintsPolicy indicates how we will treat node taints when calculating - pod topology spread skew. Options are: - - Honor: nodes without taints, along with tainted nodes for which the incoming pod - has a toleration, are included. - - Ignore: node taints are ignored. All nodes are included. - - If this value is nil, the behavior is equivalent to the Ignore policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. - type: string - topologyKey: - description: |- - TopologyKey is the key of node labels. Nodes that have a label with this key - and identical values are considered to be in the same topology. - We consider each as a "bucket", and try to put balanced number - of pods into each bucket. - We define a domain as a particular instance of a topology. - Also, we define an eligible domain as a domain whose nodes meet the requirements of - nodeAffinityPolicy and nodeTaintsPolicy. - e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. - And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. - It's a required field. - type: string - whenUnsatisfiable: - description: |- - WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy - the spread constraint. - - DoNotSchedule (default) tells the scheduler not to schedule it. - - ScheduleAnyway tells the scheduler to schedule the pod in any location, - but giving higher precedence to topologies that would help reduce the - skew. - A constraint is considered "Unsatisfiable" for an incoming pod - if and only if every possible node assignment for that pod would violate - "MaxSkew" on some topology. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 3/1/1: - | zone1 | zone2 | zone3 | - | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled - to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies - MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler - won't make it *more* imbalanced. - It's a required field. - type: string - required: - - maxSkew - - topologyKey - - whenUnsatisfiable - type: object - type: array - x-kubernetes-list-map-keys: - - topologyKey - - whenUnsatisfiable - x-kubernetes-list-type: map - volumes: - description: |- - List of volumes that can be mounted by containers belonging to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes - items: - description: Volume represents a named volume in a - pod that may be accessed by any container in the - pod. - properties: - awsElasticBlockStore: - description: |- - awsElasticBlockStore represents an AWS Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - format: int32 - type: integer - readOnly: - description: |- - readOnly value true will force the readOnly setting in VolumeMounts. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: boolean - volumeID: - description: |- - volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: string - required: - - volumeID - type: object - azureDisk: - description: azureDisk represents an Azure Data - Disk mount on the host and bind mount to the - pod. - properties: - cachingMode: - description: 'cachingMode is the Host Caching - mode: None, Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data - disk in the blob storage - type: string - diskURI: - description: diskURI is the URI of data disk - in the blob storage - type: string - fsType: - default: ext4 - description: |- - fsType is Filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - kind: - description: 'kind expected values are Shared: - multiple blob disks per storage account Dedicated: - single blob disk per storage account Managed: - azure managed data disk (only in managed - availability set). defaults to shared' - type: string - readOnly: - default: false - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: azureFile represents an Azure File - Service mount on the host and bind mount to - the pod. - properties: - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret - that contains Azure Storage Account Name - and Key - type: string - shareName: - description: shareName is the azure share - Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: cephFS represents a Ceph FS mount - on the host that shares a pod's lifetime - properties: - monitors: - description: |- - monitors is Required: Monitors is a collection of Ceph monitors - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - items: - type: string - type: array - x-kubernetes-list-type: atomic - path: - description: 'path is Optional: Used as the - mounted root, rather than the full Ceph - tree, default is /' - type: string - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: boolean - secretFile: - description: |- - secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - secretRef: - description: |- - secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - user: - description: |- - user is optional: User is the rados user name, default is admin - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - required: - - monitors - type: object - cinder: - description: |- - cinder represents a cinder volume attached and mounted on kubelets host machine. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: boolean - secretRef: - description: |- - secretRef is optional: points to a secret object containing parameters used to connect - to OpenStack. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - volumeID: - description: |- - volumeID used to identify the volume in cinder. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap - that should populate this volume - properties: - defaultMode: - description: |- - defaultMode is optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path - within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional specify whether the - ConfigMap or its keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - csi: - description: csi (Container Storage Interface) - represents ephemeral storage that is handled - by certain external CSI drivers (Beta feature). - properties: - driver: - description: |- - driver is the name of the CSI driver that handles this volume. - Consult with your admin for the correct name as registered in the cluster. - type: string - fsType: - description: |- - fsType to mount. Ex. "ext4", "xfs", "ntfs". - If not provided, the empty value is passed to the associated CSI driver - which will determine the default filesystem to apply. - type: string - nodePublishSecretRef: - description: |- - nodePublishSecretRef is a reference to the secret object containing - sensitive information to pass to the CSI driver to complete the CSI - NodePublishVolume and NodeUnpublishVolume calls. - This field is optional, and may be empty if no secret is required. If the - secret object contains more than one secret, all secret references are passed. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - readOnly: - description: |- - readOnly specifies a read-only configuration for the volume. - Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: |- - volumeAttributes stores driver-specific properties that are passed to the CSI - driver. Consult your driver's documentation for supported values. - type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API - about the pod that should populate this volume - properties: - defaultMode: - description: |- - Optional: mode bits to use on created files by default. Must be a - Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: Items is a list of downward API - volume file - items: - description: DownwardAPIVolumeFile represents - information to create the file containing - the pod field - properties: - fieldRef: - description: 'Required: Selects a field - of the pod: only annotations, labels, - name, namespace and uid are supported.' - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path is the - relative path name of the file to - be created. Must not be absolute or - contain the ''..'' path. Must be utf-8 - encoded. The first item of the relative - path must not start with ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - emptyDir: - description: |- - emptyDir represents a temporary directory that shares a pod's lifetime. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - properties: - medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: |- - ephemeral represents a volume that is handled by a cluster storage driver. - The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, - and deleted when the pod is removed. - - Use this if: - a) the volume is only needed while the pod runs, - b) features of normal volumes like restoring from snapshot or capacity - tracking are needed, - c) the storage driver is specified through a storage class, and - d) the storage driver supports dynamic volume provisioning through - a PersistentVolumeClaim (see EphemeralVolumeSource for more - information on the connection between this volume type - and PersistentVolumeClaim). - - Use PersistentVolumeClaim or one of the vendor-specific - APIs for volumes that persist for longer than the lifecycle - of an individual pod. - - Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to - be used that way - see the documentation of the driver for - more information. - - A pod can use both types of ephemeral volumes and - persistent volumes at the same time. - properties: - volumeClaimTemplate: - description: |- - Will be used to create a stand-alone PVC to provision the volume. - The pod in which this EphemeralVolumeSource is embedded will be the - owner of the PVC, i.e. the PVC will be deleted together with the - pod. The name of the PVC will be `-` where - `` is the name from the `PodSpec.Volumes` array - entry. Pod validation will reject the pod if the concatenated name - is not valid for a PVC (for example, too long). - - An existing PVC with that name that is not owned by the pod - will *not* be used for the pod to avoid using an unrelated - volume by mistake. Starting the pod is then blocked until - the unrelated PVC is removed. If such a pre-created PVC is - meant to be used by the pod, the PVC has to updated with an - owner reference to the pod once the pod exists. Normally - this should not be necessary, but it may be useful when - manually reconstructing a broken cluster. - - This field is read-only and no changes will be made by Kubernetes - to the PVC after it has been created. - - Required, must not be nil. - properties: - metadata: - description: |- - May contain labels and annotations that will be copied into the PVC - when creating it. No other fields are allowed and will be rejected during - validation. - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - The specification for the PersistentVolumeClaim. The entire content is - copied unchanged into the PVC that gets created from this - template. The same fields as in a PersistentVolumeClaim - are also valid here. - properties: - accessModes: - description: |- - accessModes contains the desired access modes the volume should have. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 - items: - type: string - type: array - x-kubernetes-list-type: atomic - dataSource: - description: |- - dataSource field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - If the provisioner or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified data source. - When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, - and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. - If the namespace is specified, then dataSourceRef will not be copied to dataSource. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type - of resource being referenced - type: string - name: - description: Name is the name - of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: |- - dataSourceRef specifies the object from which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a non-empty API group (non - core object) or a PersistentVolumeClaim object. - When this field is specified, volume binding will only succeed if the type of - the specified object matches some installed volume populator or dynamic - provisioner. - This field will replace the functionality of the dataSource field and as such - if both fields are non-empty, they must have the same value. For backwards - compatibility, when namespace isn't specified in dataSourceRef, - both fields (dataSource and dataSourceRef) will be set to the same - value automatically if one of them is empty and the other is non-empty. - When namespace is specified in dataSourceRef, - dataSource isn't set to the same value and must be empty. - There are three important differences between dataSource and dataSourceRef: - * While dataSource only allows two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping them), dataSourceRef - preserves all values, and generates an error if a disallowed value is - specified. - * While dataSource only allows local objects, dataSourceRef allows objects - in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. - (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type - of resource being referenced - type: string - name: - description: Name is the name - of resource being referenced - type: string - namespace: - description: |- - Namespace is the namespace of resource being referenced - Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. - (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: |- - resources represents the minimum resources the volume should have. - If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements - that are lower than previous value but must still be higher than capacity recorded in the - status field of the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - description: selector is a label query - over volumes to consider for binding. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: |- - storageClassName is the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 - type: string - volumeAttributesClassName: - description: |- - volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. - If specified, the CSI driver will create or update the volume with the attributes defined - in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. - If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be - set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource - exists. - More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). - type: string - volumeMode: - description: |- - volumeMode defines what type of volume is required by the claim. - Value of Filesystem is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding - reference to the PersistentVolume - backing this claim. - type: string - type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource - that is attached to a kubelet's host machine - and then exposed to the pod. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - lun: - description: 'lun is Optional: FC target lun - number' - format: int32 - type: integer - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target - worldwide names (WWNs)' - items: - type: string - type: array - x-kubernetes-list-type: atomic - wwids: - description: |- - wwids Optional: FC volume world wide identifiers (wwids) - Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - flexVolume: - description: |- - flexVolume represents a generic volume resource that is - provisioned/attached using an exec based plugin. - properties: - driver: - description: driver is the name of the driver - to use for this volume. - type: string - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field - holds extra command options if any.' - type: object - readOnly: - description: |- - readOnly is Optional: defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef is Optional: secretRef is reference to the secret object containing - sensitive information to pass to the plugin scripts. This may be - empty if no secret object is specified. If the secret object - contains more than one secret, all secrets are passed to the plugin - scripts. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - required: - - driver - type: object - flocker: - description: flocker represents a Flocker volume - attached to a kubelet's host machine. This depends - on the Flocker control service being running - properties: - datasetName: - description: |- - datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker - should be considered as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the - dataset. This is unique identifier of a - Flocker dataset - type: string - type: object - gcePersistentDisk: - description: |- - gcePersistentDisk represents a GCE Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - properties: - fsType: - description: |- - fsType is filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - format: int32 - type: integer - pdName: - description: |- - pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: boolean - required: - - pdName - type: object - gitRepo: - description: |- - gitRepo represents a git repository at a particular revision. - DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an - EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir - into the Pod's container. - properties: - directory: - description: |- - directory is the target directory name. - Must not contain or start with '..'. If '.' is supplied, the volume directory will be the - git repository. Otherwise, if specified, the volume will contain the git repository in - the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for - the specified revision. - type: string - required: - - repository - type: object - glusterfs: - description: |- - glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. - More info: https://examples.k8s.io/volumes/glusterfs/README.md - properties: - endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: string - path: - description: |- - path is the Glusterfs volume path. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: string - readOnly: - description: |- - readOnly here will force the Glusterfs volume to be mounted with read-only permissions. - Defaults to false. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: boolean - required: - - endpoints - - path - type: object - hostPath: - description: |- - hostPath represents a pre-existing file or directory on the host - machine that is directly exposed to the container. This is generally - used for system agents or other privileged things that are allowed - to see the host machine. Most containers will NOT need this. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - properties: - path: - description: |- - path of the directory on the host. - If the path is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - type: - description: |- - type for HostPath Volume - Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - required: - - path - type: object - image: - description: |- - image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. - The volume is resolved at pod startup depending on which PullPolicy value is provided: - - - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - - The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. - A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. - The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. - The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. - The volume will be mounted read-only (ro) and non-executable files (noexec). - Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). - The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. - properties: - pullPolicy: - description: |- - Policy for pulling OCI objects. Possible values are: - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - type: string - reference: - description: |- - Required: Image or artifact reference to be used. - Behaves in the same way as pod.spec.containers[*].image. - Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - type: object - iscsi: - description: |- - iscsi represents an ISCSI Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md - properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether - support iSCSI Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether - support iSCSI Session CHAP authentication - type: boolean - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - type: string - initiatorName: - description: |- - initiatorName is the custom iSCSI Initiator Name. - If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface - : will be created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified - Name. - type: string - iscsiInterface: - default: default - description: |- - iscsiInterface is the interface Name that uses an iSCSI transport. - Defaults to 'default' (tcp). - type: string - lun: - description: lun represents iSCSI Target Lun - number. - format: int32 - type: integer - portals: - description: |- - portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - items: - type: string - type: array - x-kubernetes-list-type: atomic - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret - for iSCSI target and initiator authentication - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - targetPortal: - description: |- - targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - description: |- - name of the volume. - Must be a DNS_LABEL and unique within the pod. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - nfs: - description: |- - nfs represents an NFS mount on the host that shares a pod's lifetime - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - properties: - path: - description: |- - path that is exported by the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - readOnly: - description: |- - readOnly here will force the NFS export to be mounted with read-only permissions. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: boolean - server: - description: |- - server is the hostname or IP address of the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: |- - persistentVolumeClaimVolumeSource represents a reference to a - PersistentVolumeClaim in the same namespace. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - properties: - claimName: - description: |- - claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - type: string - readOnly: - description: |- - readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: photonPersistentDisk represents a - PhotonController persistent disk attached and - mounted on kubelets host machine - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - pdID: - description: pdID is the ID that identifies - Photon Controller persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: portworxVolume represents a portworx - volume attached and mounted on kubelets host - machine - properties: - fsType: - description: |- - fSType represents the filesystem type to mount - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies - a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources - secrets, configmaps, and downward API - properties: - defaultMode: - description: |- - defaultMode are the mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - sources: - description: |- - sources is the list of volume projections. Each entry in this list - handles one source. - items: - description: |- - Projection that may be projected along with other supported volume types. - Exactly one of these fields must be set. - properties: - clusterTrustBundle: - description: |- - ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field - of ClusterTrustBundle objects in an auto-updating file. - - Alpha, gated by the ClusterTrustBundleProjection feature gate. - - ClusterTrustBundle objects can either be selected by name, or by the - combination of signer name and a label selector. - - Kubelet performs aggressive normalization of the PEM contents written - into the pod filesystem. Esoteric PEM features such as inter-block - comments and block headers are stripped. Certificates are deduplicated. - The ordering of certificates within the file is arbitrary, and Kubelet - may change the order over time. - properties: - labelSelector: - description: |- - Select all ClusterTrustBundles that match this label selector. Only has - effect if signerName is set. Mutually-exclusive with name. If unset, - interpreted as "match nothing". If set but empty, interpreted as "match - everything". - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - name: - description: |- - Select a single ClusterTrustBundle by object name. Mutually-exclusive - with signerName and labelSelector. - type: string - optional: - description: |- - If true, don't block pod startup if the referenced ClusterTrustBundle(s) - aren't available. If using name, then the named ClusterTrustBundle is - allowed not to exist. If using signerName, then the combination of - signerName and labelSelector is allowed to match zero - ClusterTrustBundles. - type: boolean - path: - description: Relative path from - the volume root to write the bundle. - type: string - signerName: - description: |- - Select all ClusterTrustBundles that match this signer name. - Mutually-exclusive with name. The contents of all selected - ClusterTrustBundles will be unified and deduplicated. - type: string - required: - - path - type: object - configMap: - description: configMap information about - the configMap data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key - to a path within a volume. - properties: - key: - description: key is the key - to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional specify whether - the ConfigMap or its keys must - be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - downwardAPI: - description: downwardAPI information - about the downwardAPI data to project - properties: - items: - description: Items is a list of - DownwardAPIVolume file - items: - description: DownwardAPIVolumeFile - represents information to create - the file containing the pod - field - properties: - fieldRef: - description: 'Required: Selects - a field of the pod: only - annotations, labels, name, - namespace and uid are supported.' - properties: - apiVersion: - description: Version of - the schema the FieldPath - is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the - field to select in the - specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path - is the relative path name - of the file to be created. - Must not be absolute or - contain the ''..'' path. - Must be utf-8 encoded. The - first item of the relative - path must not start with - ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container - name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies - the output format of - the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: - resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - secret: - description: secret information about - the secret data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key - to a path within a volume. - properties: - key: - description: key is the key - to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional field specify - whether the Secret or its key - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - serviceAccountToken: - description: serviceAccountToken is - information about the serviceAccountToken - data to project - properties: - audience: - description: |- - audience is the intended audience of the token. A recipient of a token - must identify itself with an identifier specified in the audience of the - token, and otherwise should reject the token. The audience defaults to the - identifier of the apiserver. - type: string - expirationSeconds: - description: |- - expirationSeconds is the requested duration of validity of the service - account token. As the token approaches expiration, the kubelet volume - plugin will proactively rotate the service account token. The kubelet will - start trying to rotate the token if the token is older than 80 percent of - its time to live or if the token is older than 24 hours.Defaults to 1 hour - and must be at least 10 minutes. - format: int64 - type: integer - path: - description: |- - path is the path relative to the mount point of the file to project the - token into. - type: string - required: - - path - type: object - type: object - type: array - x-kubernetes-list-type: atomic - type: object - quobyte: - description: quobyte represents a Quobyte mount - on the host that shares a pod's lifetime - properties: - group: - description: |- - group to map volume access to - Default is no group - type: string - readOnly: - description: |- - readOnly here will force the Quobyte volume to be mounted with read-only permissions. - Defaults to false. - type: boolean - registry: - description: |- - registry represents a single or multiple Quobyte Registry services - specified as a string as host:port pair (multiple entries are separated with commas) - which acts as the central registry for volumes - type: string - tenant: - description: |- - tenant owning the given Quobyte volume in the Backend - Used with dynamically provisioned Quobyte volumes, value is set by the plugin - type: string - user: - description: |- - user to map volume access to - Defaults to serivceaccount user - type: string - volume: - description: volume is a string that references - an already created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: |- - rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. - More info: https://examples.k8s.io/volumes/rbd/README.md - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - type: string - image: - description: |- - image is the rados image name. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - keyring: - default: /etc/ceph/keyring - description: |- - keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - monitors: - description: |- - monitors is a collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - items: - type: string - type: array - x-kubernetes-list-type: atomic - pool: - default: rbd - description: |- - pool is the rados pool name. - Default is rbd. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: boolean - secretRef: - description: |- - secretRef is name of the authentication secret for RBDUser. If provided - overrides keyring. - Default is nil. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - user: - default: admin - description: |- - user is the rados user name. - Default is admin. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - required: - - image - - monitors - type: object - scaleIO: - description: scaleIO represents a ScaleIO persistent - volume attached and mounted on Kubernetes nodes. - properties: - fsType: - default: xfs - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". - Default is "xfs". - type: string - gateway: - description: gateway is the host address of - the ScaleIO API Gateway. - type: string - protectionDomain: - description: protectionDomain is the name - of the ScaleIO Protection Domain for the - configured storage. - type: string - readOnly: - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef references to the secret for ScaleIO user and other - sensitive information. If this is not provided, Login operation will fail. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - sslEnabled: - description: sslEnabled Flag enable/disable - SSL communication with Gateway, default - false - type: boolean - storageMode: - default: ThinProvisioned - description: |- - storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage - Pool associated with the protection domain. - type: string - system: - description: system is the name of the storage - system as configured in ScaleIO. - type: string - volumeName: - description: |- - volumeName is the name of a volume already created in the ScaleIO system - that is associated with this volume source. - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - description: |- - secret represents a secret that should populate this volume. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - properties: - defaultMode: - description: |- - defaultMode is Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values - for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items If unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path - within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - optional: - description: optional field specify whether - the Secret or its keys must be defined - type: boolean - secretName: - description: |- - secretName is the name of the secret in the pod's namespace to use. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - type: string - type: object - storageos: - description: storageOS represents a StorageOS - volume attached and mounted on Kubernetes nodes. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef specifies the secret to use for obtaining the StorageOS API - credentials. If not specified, default values will be attempted. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - volumeName: - description: |- - volumeName is the human-readable name of the StorageOS volume. Volume - names are only unique within a namespace. - type: string - volumeNamespace: - description: |- - volumeNamespace specifies the scope of the volume within StorageOS. If no - namespace is specified then the Pod's namespace will be used. This allows the - Kubernetes name scoping to be mirrored within StorageOS for tighter integration. - Set VolumeName to any name to override the default behaviour. - Set to "default" if you are not using namespaces within StorageOS. - Namespaces that do not pre-exist within StorageOS will be created. - type: string - type: object - vsphereVolume: - description: vsphereVolume represents a vSphere - volume attached and mounted on kubelets host - machine - properties: - fsType: - description: |- - fsType is filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage - Policy Based Management (SPBM) profile ID - associated with the StoragePolicyName. - type: string - storagePolicyName: - description: storagePolicyName is the storage - Policy Based Management (SPBM) profile name. - type: string - volumePath: - description: volumePath is the path that identifies - vSphere volume vmdk - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - containers - type: object - type: object - topologyRequest: - description: topologyRequest defines the topology request for - the PodSet. - properties: - podIndexLabel: - description: |- - PodIndexLabel indicates the name of the label indexing the pods. - For example, in the context of - - kubernetes job this is: kubernetes.io/job-completion-index - - JobSet: kubernetes.io/job-completion-index (inherited from Job) - - Kubeflow: training.kubeflow.org/replica-index - type: string - preferred: - description: |- - preferred indicates the topology level preferred by the PodSet, as - indicated by the `kueue.x-k8s.io/podset-preferred-topology` PodSet - annotation. - type: string - required: - description: |- - required indicates the topology level required by the PodSet, as - indicated by the `kueue.x-k8s.io/podset-required-topology` PodSet - annotation. - type: string - subGroupCount: - description: |- - SubGroupIndexLabel indicates the count of replicated Jobs (groups) within a PodSet. - For example, in the context of JobSet this value is read from jobset.sigs.k8s.io/replicatedjob-replicas. - format: int32 - type: integer - subGroupIndexLabel: - description: |- - SubGroupIndexLabel indicates the name of the label indexing the instances of replicated Jobs (groups) - within a PodSet. For example, in the context of JobSet this is jobset.sigs.k8s.io/job-index. - type: string - type: object - required: - - count - - template - type: object - x-kubernetes-validations: - - message: minCount should be positive and less or equal to count - rule: 'has(self.minCount) ? self.minCount <= self.count : true' - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - priority: - description: |- - Priority determines the order of access to the resources managed by the - ClusterQueue where the workload is queued. - The priority value is populated from PriorityClassName. - The higher the value, the higher the priority. - If priorityClassName is specified, priority must not be null. - format: int32 - type: integer - priorityClassName: - description: |- - If specified, indicates the workload's priority. - "system-node-critical" and "system-cluster-critical" are two special - keywords which indicate the highest priorities with the former being - the highest priority. Any other name must be defined by creating a - PriorityClass object with that name. If not specified, the workload - priority will be default or zero if there is no default. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - priorityClassSource: - default: "" - description: |- - priorityClassSource determines whether the priorityClass field refers to a pod PriorityClass or kueue.x-k8s.io/workloadpriorityclass. - Workload's PriorityClass can accept the name of a pod priorityClass or a workloadPriorityClass. - When using pod PriorityClass, a priorityClassSource field has the scheduling.k8s.io/priorityclass value. - enum: - - kueue.x-k8s.io/workloadpriorityclass - - scheduling.k8s.io/priorityclass - - "" - type: string - queueName: - description: |- - queueName is the name of the LocalQueue the Workload is associated with. - queueName cannot be changed while .status.admission is not null. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - podSets - type: object - x-kubernetes-validations: - - message: priority should not be nil when priorityClassName is set - rule: 'has(self.priorityClassName) ? has(self.priority) : true' - status: - description: WorkloadStatus defines the observed state of Workload - properties: - accumulatedPastExexcutionTimeSeconds: - description: |- - accumulatedPastExexcutionTimeSeconds holds the total time, in seconds, the workload spent - in Admitted state, in the previous `Admit` - `Evict` cycles. - format: int32 - type: integer - admission: - description: |- - admission holds the parameters of the admission of the workload by a - ClusterQueue. admission can be set back to null, but its fields cannot be - changed once set. - properties: - clusterQueue: - description: clusterQueue is the name of the ClusterQueue that - admitted this workload. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - podSetAssignments: - description: PodSetAssignments hold the admission results for - each of the .spec.podSets entries. - items: - properties: - count: - description: |- - count is the number of pods taken into account at admission time. - This field will not change in case of quota reclaim. - Value could be missing for Workloads created before this field was added, - in that case spec.podSets[*].count value will be used. - format: int32 - minimum: 0 - type: integer - flavors: - additionalProperties: - description: ResourceFlavorReference is the name of the - ResourceFlavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - description: Flavors are the flavors assigned to the workload - for each resource. - type: object - name: - default: main - description: Name is the name of the podSet. It should match - one of the names in .spec.podSets. - maxLength: 63 - pattern: ^(?i)[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - resourceUsage: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - resourceUsage keeps track of the total resources all the pods in the podset need to run. - - Beside what is provided in podSet's specs, this calculation takes into account - the LimitRange defaults and RuntimeClass overheads at the moment of admission. - This field will not change in case of quota reclaim. - type: object - topologyAssignment: - description: |- - topologyAssignment indicates the topology assignment divided into - topology domains corresponding to the lowest level of the topology. - The assignment specifies the number of Pods to be scheduled per topology - domain and specifies the node selectors for each topology domain, in the - following way: the node selector keys are specified by the levels field - (same for all domains), and the corresponding node selector value is - specified by the domains.values subfield. If the TopologySpec.Levels field contains - "kubernetes.io/hostname" label, topologyAssignment will contain data only for - this label, and omit higher levels in the topology - - Example: - - topologyAssignment: - levels: - - cloud.provider.com/topology-block - - cloud.provider.com/topology-rack - domains: - - values: [block-1, rack-1] - count: 4 - - values: [block-1, rack-2] - count: 2 - - Here: - - 4 Pods are to be scheduled on nodes matching the node selector: - cloud.provider.com/topology-block: block-1 - cloud.provider.com/topology-rack: rack-1 - - 2 Pods are to be scheduled on nodes matching the node selector: - cloud.provider.com/topology-block: block-1 - cloud.provider.com/topology-rack: rack-2 - - Example: - Below there is an equivalent of the above example assuming, Topology - object defines kubernetes.io/hostname as the lowest level in topology. - Hence we omit higher level of topologies, since the hostname label - is sufficient to explicitly identify a proper node. - - topologyAssignment: - levels: - - kubernetes.io/hostname - domains: - - values: [hostname-1] - count: 4 - - values: [hostname-2] - count: 2 - properties: - domains: - description: |- - domains is a list of topology assignments split by topology domains at - the lowest level of the topology. - items: - properties: - count: - description: |- - count indicates the number of Pods to be scheduled in the topology - domain indicated by the values field. - format: int32 - minimum: 1 - type: integer - values: - description: |- - values is an ordered list of node selector values describing a topology - domain. The values correspond to the consecutive topology levels, from - the highest to the lowest. - items: - type: string - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - required: - - count - - values - type: object - type: array - levels: - description: |- - levels is an ordered list of keys denoting the levels of the assigned - topology (i.e. node label keys), from the highest to the lowest level of - the topology. - items: - type: string - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - required: - - domains - - levels - type: object - required: - - name - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - clusterQueue - - podSetAssignments - type: object - admissionChecks: - description: admissionChecks list all the admission checks required - by the workload and the current status - items: - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - name: - description: name identifies the admission check. - maxLength: 316 - type: string - podSetUpdates: - items: - description: |- - PodSetUpdate contains a list of pod set modifications suggested by AdmissionChecks. - The modifications should be additive only - modifications of already existing keys - or having the same key provided by multiple AdmissionChecks is not allowed and will - result in failure during workload admission. - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - name: - description: Name of the PodSet to modify. Should match - to one of the Workload's PodSets. - type: string - nodeSelector: - additionalProperties: - type: string - type: object - tolerations: - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - maxItems: 8 - type: array - x-kubernetes-validations: - - message: operator must be Exists when 'key' is empty, - which means 'match all values and all keys' - rule: 'self.all(x, !has(x.key) ? x.operator == ''Exists'' - : true)' - - message: effect must be 'NoExecute' when 'tolerationSeconds' - is set - rule: 'self.all(x, has(x.tolerationSeconds) ? x.effect - == ''NoExecute'' : true)' - - message: 'supported toleration values: ''Equal''(default), - ''Exists''' - rule: self.all(x, !has(x.operator) || x.operator in - ['Equal', 'Exists']) - - message: a value must be empty when 'operator' is 'Exists' - rule: 'self.all(x, has(x.operator) && x.operator == - ''Exists'' ? !has(x.value) : true)' - - message: 'supported taint effect values: ''NoSchedule'', - ''PreferNoSchedule'', ''NoExecute''' - rule: self.all(x, !has(x.effect) || x.effect in ['NoSchedule', - 'PreferNoSchedule', 'NoExecute']) - required: - - name - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - state: - description: state of the admissionCheck, one of Pending, Ready, - Retry, Rejected - enum: - - Pending - - Ready - - Retry - - Rejected - type: string - required: - - lastTransitionTime - - message - - name - - state - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - conditions: - description: |- - conditions hold the latest available observations of the Workload - current state. - - The type of the condition could be: - - - Admitted: the Workload was admitted through a ClusterQueue. - - Finished: the associated workload finished running (failed or succeeded). - - PodsReady: at least `.spec.podSets[*].count` Pods are ready or have - succeeded. - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - reclaimablePods: - description: |- - reclaimablePods keeps track of the number pods within a podset for which - the resource reservation is no longer needed. - items: - properties: - count: - description: count is the number of pods for which the requested - resources are no longer needed. - format: int32 - minimum: 0 - type: integer - name: - description: name is the PodSet name. - type: string - required: - - count - - name - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - requeueState: - description: |- - requeueState holds the re-queue state - when a workload meets Eviction with PodsReadyTimeout reason. - properties: - count: - description: |- - count records the number of times a workload has been re-queued - When a deactivated (`.spec.activate`=`false`) workload is reactivated (`.spec.activate`=`true`), - this count would be reset to null. - format: int32 - minimum: 0 - type: integer - requeueAt: - description: |- - requeueAt records the time when a workload will be re-queued. - When a deactivated (`.spec.activate`=`false`) workload is reactivated (`.spec.activate`=`true`), - this time would be reset to null. - format: date-time - type: string - type: object - resourceRequests: - description: |- - resourceRequests provides a detailed view of the resources that were - requested by a non-admitted workload when it was considered for admission. - If admission is non-null, resourceRequests will be empty because - admission.resourceUsage contains the detailed information. - items: - properties: - name: - default: main - description: name is the name of the podSet. It should match - one of the names in .spec.podSets. - maxLength: 63 - pattern: ^(?i)[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - resources: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - resources is the total resources all the pods in the podset need to run. - - Beside what is provided in podSet's specs, this value also takes into account - the LimitRange defaults and RuntimeClass overheads at the moment of consideration - and the application of resource.excludeResourcePrefixes and resource.transformations. - type: object - required: - - name - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - type: object - x-kubernetes-validations: - - message: podSetAssignments must have the same number of podSets as the spec - rule: 'has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'') && has(self.status.admission) - ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) - : true' - - message: field is immutable - rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'')) ? (oldSelf.spec.priorityClassSource - == self.spec.priorityClassSource) : true' - - message: field is immutable - rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassName) - && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName - == self.spec.priorityClassName) : true' - - message: field is immutable - rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'')) && (has(self.status) - && has(self.status.conditions) && self.status.conditions.exists(c, c.type - == ''QuotaReserved'' && c.status == ''True'')) && has(oldSelf.spec.queueName) - && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName - : true' - - message: maximumExecutionTimeSeconds is immutable while admitted - rule: ((has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == 'Admitted' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) - && self.status.conditions.exists(c, c.type == 'Admitted' && c.status == - 'True')))?((has(oldSelf.spec.maximumExecutionTimeSeconds)?oldSelf.spec.maximumExecutionTimeSeconds:0) - == (has(self.spec.maximumExecutionTimeSeconds)?self.spec.maximumExecutionTimeSeconds:0)):true - served: true - storage: true - subresources: - status: {} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-controller-manager - namespace: kueue-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-leader-election-role - namespace: kueue-system -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch ---- -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.kueue.x-k8s.io/batch-admin: "true" -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-batch-admin-role ---- -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.kueue.x-k8s.io/batch-user: "true" -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-batch-user-role ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-clusterqueue-editor-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - clusterqueues - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - clusterqueues/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-clusterqueue-viewer-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - clusterqueues - verbs: - - get - - list - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - clusterqueues/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-job-editor-role -rules: -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-job-viewer-role -rules: -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch -- apiGroups: - - batch - resources: - - jobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-jobset-editor-role -rules: -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-jobset-viewer-role -rules: -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets - verbs: - - get - - list - - watch -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-localqueue-editor-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - localqueues - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - localqueues/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-localqueue-viewer-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - localqueues - verbs: - - get - - list - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - localqueues/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-manager-role -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - watch -- apiGroups: - - "" - resources: - - limitranges - - namespaces - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods/finalizers - verbs: - - get - - update -- apiGroups: - - "" - resources: - - pods/status - verbs: - - get - - patch -- apiGroups: - - "" - resources: - - podtemplates - verbs: - - create - - delete - - get - - list - - update - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - update - - watch -- apiGroups: - - admissionregistration.k8s.io - resources: - - mutatingwebhookconfigurations - - validatingwebhookconfigurations - verbs: - - get - - list - - update - - watch -- apiGroups: - - admissionregistration.k8s.io - resources: - - validatingadmissionpolicies - - validatingadmissionpolicybindings - verbs: - - get - - list - - watch -- apiGroups: - - apps - resources: - - replicasets - - statefulsets - verbs: - - get - - list - - watch -- apiGroups: - - autoscaling.x-k8s.io - resources: - - provisioningrequests - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - autoscaling.x-k8s.io - resources: - - provisioningrequests/status - verbs: - - get -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs/finalizers - - jobs/status - verbs: - - get - - patch - - update -- apiGroups: - - flowcontrol.apiserver.k8s.io - resources: - - flowschemas - - prioritylevelconfigurations - verbs: - - list - - watch -- apiGroups: - - flowcontrol.apiserver.k8s.io - resources: - - flowschemas/status - verbs: - - patch -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/finalizers - verbs: - - get - - update -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/status - verbs: - - get - - patch - - update -- apiGroups: - - kubeflow.org - resources: - - mpijobs - - mxjobs - - paddlejobs - - pytorchjobs - - tfjobs - - xgboostjobs - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - mpijobs/finalizers - - mxjobs/finalizers - - mxjobs/status - - paddlejobs/finalizers - - pytorchjobs/finalizers - - tfjobs/finalizers - - xgboostjobs/finalizers - verbs: - - get - - update -- apiGroups: - - kubeflow.org - resources: - - mpijobs/status - - paddlejobs/status - - pytorchjobs/status - - tfjobs/status - - xgboostjobs/status - verbs: - - get - - patch - - update -- apiGroups: - - kueue.x-k8s.io - resources: - - admissionchecks - - clusterqueues - - cohorts - - localqueues - - workloads - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - admissionchecks/finalizers - - clusterqueues/finalizers - - localqueues/finalizers - - resourceflavors/finalizers - - topology/finalizers - - workloads/finalizers - verbs: - - update -- apiGroups: - - kueue.x-k8s.io - resources: - - admissionchecks/status - - clusterqueues/status - - localqueues/status - - multikueueclusters/status - - workloads/status - verbs: - - get - - patch - - update -- apiGroups: - - kueue.x-k8s.io - resources: - - multikueueclusters - - multikueueconfigs - - provisioningrequestconfigs - - topologies - - workloadpriorityclasses - verbs: - - get - - list - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - resourceflavors - verbs: - - delete - - get - - list - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - topology - verbs: - - get - - list - - update - - watch -- apiGroups: - - node.k8s.io - resources: - - runtimeclasses - verbs: - - get - - list - - watch -- apiGroups: - - ray.io - resources: - - rayclusters - - rayjobs - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - ray.io - resources: - - rayclusters/finalizers - - rayclusters/status - - rayjobs/finalizers - - rayjobs/status - verbs: - - get - - update -- apiGroups: - - scheduling.k8s.io - resources: - - priorityclasses - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-metrics-reader -rules: -- nonResourceURLs: - - /metrics - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-mpijob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - mpijobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - mpijobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-mpijob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - mpijobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - mpijobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-mxjob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - mxjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - mxjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-mxjob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - mxjobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - mxjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-paddlejob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - paddlejobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - paddlejobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-paddlejob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - paddlejobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - paddlejobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-pending-workloads-cq-viewer-role -rules: -- apiGroups: - - visibility.kueue.x-k8s.io - resources: - - clusterqueues/pendingworkloads - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-pending-workloads-lq-viewer-role -rules: -- apiGroups: - - visibility.kueue.x-k8s.io - resources: - - localqueues/pendingworkloads - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-pytorchjob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-pytorchjob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-raycluster-editor-role -rules: -- apiGroups: - - ray.io - resources: - - rayclusters - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - ray.io - resources: - - rayclusters/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-raycluster-viewer-role -rules: -- apiGroups: - - ray.io - resources: - - rayclusters - verbs: - - get - - list - - watch -- apiGroups: - - ray.io - resources: - - rayclusters/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-rayjob-editor-role -rules: -- apiGroups: - - ray.io - resources: - - rayjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - ray.io - resources: - - rayjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-rayjob-viewer-role -rules: -- apiGroups: - - ray.io - resources: - - rayjobs - verbs: - - get - - list - - watch -- apiGroups: - - ray.io - resources: - - rayjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-resourceflavor-editor-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - resourceflavors - verbs: - - create - - delete - - get - - list - - patch - - update - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-resourceflavor-viewer-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - resourceflavors - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-tfjob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - tfjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - tfjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-tfjob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - tfjobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - tfjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-workload-editor-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - workloads - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - workloads/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-workload-viewer-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - workloads - verbs: - - get - - list - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - workloads/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-xgboostjob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - xgboostjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - xgboostjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-xgboostjob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - xgboostjobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - xgboostjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-visibility-server-auth-reader - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader -subjects: -- kind: ServiceAccount - name: kueue-controller-manager - namespace: kueue-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-leader-election-rolebinding - namespace: kueue-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: kueue-leader-election-role -subjects: -- kind: ServiceAccount - name: kueue-controller-manager - namespace: kueue-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kueue-manager-role -subjects: -- kind: ServiceAccount - name: kueue-controller-manager - namespace: kueue-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kueue-proxy-role -subjects: -- kind: ServiceAccount - name: kueue-controller-manager - namespace: kueue-system ---- -apiVersion: v1 -data: - controller_manager_config.yaml: | - apiVersion: config.kueue.x-k8s.io/v1beta1 - kind: Configuration - health: - healthProbeBindAddress: :8081 - metrics: - bindAddress: :8080 - # enableClusterQueueResources: true - webhook: - port: 9443 - leaderElection: - leaderElect: true - resourceName: c1f6bfd2.kueue.x-k8s.io - controller: - groupKindConcurrency: - Job.batch: 5 - Pod: 5 - Workload.kueue.x-k8s.io: 5 - LocalQueue.kueue.x-k8s.io: 1 - Cohort.kueue.x-k8s.io: 1 - ClusterQueue.kueue.x-k8s.io: 1 - ResourceFlavor.kueue.x-k8s.io: 1 - clientConnection: - qps: 50 - burst: 100 - #pprofBindAddress: :8083 - #waitForPodsReady: - # enable: false - # timeout: 5m - # blockAdmission: false - # requeuingStrategy: - # timestamp: Eviction - # backoffLimitCount: null # null indicates infinite requeuing - # backoffBaseSeconds: 60 - # backoffMaxSeconds: 3600 - #manageJobsWithoutQueueName: true - #managedJobsNamespaceSelector: - # matchLabels: - # kueue-managed: "true" - #internalCertManagement: - # enable: false - # webhookServiceName: "" - # webhookSecretName: "" - integrations: - frameworks: - - "batch/job" - - "kubeflow.org/mpijob" - - "ray.io/rayjob" - - "ray.io/raycluster" - - "jobset.x-k8s.io/jobset" - - "kubeflow.org/mxjob" - - "kubeflow.org/paddlejob" - - "kubeflow.org/pytorchjob" - - "kubeflow.org/tfjob" - - "kubeflow.org/xgboostjob" - # - "pod" - # - "deployment" # requires enabling pod integration - # - "statefulset" # requires enabling pod integration - # externalFrameworks: - # - "Foo.v1.example.com" - # podOptions: - # namespaceSelector: - # matchExpressions: - # - key: kubernetes.io/metadata.name - # operator: NotIn - # values: [ kube-system, kueue-system ] - #fairSharing: - # enable: true - # preemptionStrategies: [LessThanOrEqualToFinalShare, LessThanInitialShare] - #resources: - # excludeResourcePrefixes: [] - # transformations: - # - input: nvidia.com/mig-4g.5gb - # strategy: Replace | Retain - # outputs: - # example.com/accelerator-memory: 5Gi - # example.com/accelerator-gpc: 4 -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-manager-config - namespace: kueue-system ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-webhook-server-cert - namespace: kueue-system ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-controller-manager-metrics-service - namespace: kueue-system -spec: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https - selector: - control-plane: controller-manager ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-visibility-server - namespace: kueue-system -spec: - ports: - - name: https - port: 443 - protocol: TCP - targetPort: 8082 - selector: - control-plane: controller-manager ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-webhook-service - namespace: kueue-system -spec: - ports: - - port: 443 - protocol: TCP - targetPort: 9443 - selector: - control-plane: controller-manager ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-controller-manager - namespace: kueue-system -spec: - replicas: 1 - selector: - matchLabels: - control-plane: controller-manager - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: manager - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - spec: - containers: - - args: - - --config=/controller_manager_config.yaml - - --zap-log-level=2 - - --feature-gates=TopologyAwareScheduling=true - command: - - /manager - image: registry.k8s.io/kueue/kueue:v0.10.0 - imagePullPolicy: Always - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - ports: - - containerPort: 8082 - name: visibility - protocol: TCP - - containerPort: 9443 - name: webhook-server - protocol: TCP - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 500m - memory: 512Mi - securityContext: - allowPrivilegeEscalation: false - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true - - mountPath: /controller_manager_config.yaml - name: manager-config - subPath: controller_manager_config.yaml - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=10 - image: registry.k8s.io/kubebuilder/kube-rbac-proxy:v0.16.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - securityContext: - runAsNonRoot: true - serviceAccountName: kueue-controller-manager - terminationGracePeriodSeconds: 10 - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: kueue-webhook-server-cert - - configMap: - name: kueue-manager-config - name: manager-config - tolerations: - - effect: NoSchedule - key: components.gke.io/gke-managed-components - operator: Equal - value: "true" ---- -apiVersion: apiregistration.k8s.io/v1 -kind: APIService -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: v1beta1.visibility.kueue.x-k8s.io -spec: - group: visibility.kueue.x-k8s.io - groupPriorityMinimum: 100 - insecureSkipTLSVerify: true - service: - name: kueue-visibility-server - namespace: kueue-system - version: v1beta1 - versionPriority: 100 ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-mutating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate--v1-pod - failurePolicy: Fail - name: mpod.kb.io - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kueue-system - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-apps-v1-deployment - failurePolicy: Fail - name: mdeployment.kb.io - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kueue-system - rules: - - apiGroups: - - apps - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - deployments - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-batch-v1-job - failurePolicy: Fail - name: mjob.kb.io - rules: - - apiGroups: - - batch - apiVersions: - - v1 - operations: - - CREATE - resources: - - jobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-jobset-x-k8s-io-v1alpha2-jobset - failurePolicy: Fail - name: mjobset.kb.io - rules: - - apiGroups: - - jobset.x-k8s.io - apiVersions: - - v1alpha2 - operations: - - CREATE - resources: - - jobsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v1-mxjob - failurePolicy: Fail - name: mmxjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - resources: - - mxjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v1-paddlejob - failurePolicy: Fail - name: mpaddlejob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - resources: - - paddlejobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v1-pytorchjob - failurePolicy: Fail - name: mpytorchjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - resources: - - pytorchjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v1-tfjob - failurePolicy: Fail - name: mtfjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - resources: - - tfjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v1-xgboostjob - failurePolicy: Fail - name: mxgboostjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - resources: - - xgboostjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v2beta1-mpijob - failurePolicy: Fail - name: mmpijob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v2beta1 - operations: - - CREATE - resources: - - mpijobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-ray-io-v1-raycluster - failurePolicy: Fail - name: mraycluster.kb.io - rules: - - apiGroups: - - ray.io - apiVersions: - - v1 - operations: - - CREATE - resources: - - rayclusters - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-ray-io-v1-rayjob - failurePolicy: Fail - name: mrayjob.kb.io - rules: - - apiGroups: - - ray.io - apiVersions: - - v1 - operations: - - CREATE - resources: - - rayjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-apps-v1-statefulset - failurePolicy: Fail - name: mstatefulset.kb.io - rules: - - apiGroups: - - apps - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - statefulsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kueue-x-k8s-io-v1beta1-clusterqueue - failurePolicy: Fail - name: mclusterqueue.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - resources: - - clusterqueues - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kueue-x-k8s-io-v1beta1-resourceflavor - failurePolicy: Fail - name: mresourceflavor.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - resources: - - resourceflavors - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kueue-x-k8s-io-v1beta1-workload - failurePolicy: Fail - name: mworkload.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - resources: - - workloads - sideEffects: None ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-validating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate--v1-pod - failurePolicy: Fail - name: vpod.kb.io - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kueue-system - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - pods - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-apps-v1-deployment - failurePolicy: Fail - name: vdeployment.kb.io - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kueue-system - rules: - - apiGroups: - - apps - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - deployments - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-batch-v1-job - failurePolicy: Fail - name: vjob.kb.io - rules: - - apiGroups: - - batch - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - jobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-jobset-x-k8s-io-v1alpha2-jobset - failurePolicy: Fail - name: vjobset.kb.io - rules: - - apiGroups: - - jobset.x-k8s.io - apiVersions: - - v1alpha2 - operations: - - CREATE - - UPDATE - resources: - - jobsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v1-mxjob - failurePolicy: Fail - name: vmxjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - mxjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v1-paddlejob - failurePolicy: Fail - name: vpaddlejob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - paddlejobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v1-pytorchjob - failurePolicy: Fail - name: vpytorchjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - pytorchjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v1-tfjob - failurePolicy: Fail - name: vtfjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - tfjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v1-xgboostjob - failurePolicy: Fail - name: vxgboostjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - xgboostjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v2beta1-mpijob - failurePolicy: Fail - name: vmpijob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v2beta1 - operations: - - CREATE - - UPDATE - resources: - - mpijobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-ray-io-v1-raycluster - failurePolicy: Fail - name: vraycluster.kb.io - rules: - - apiGroups: - - ray.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - rayclusters - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-ray-io-v1-rayjob - failurePolicy: Fail - name: vrayjob.kb.io - rules: - - apiGroups: - - ray.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - rayjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-apps-v1-statefulset - failurePolicy: Fail - name: vstatefulset.kb.io - rules: - - apiGroups: - - apps - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - statefulsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kueue-x-k8s-io-v1beta1-clusterqueue - failurePolicy: Fail - name: vclusterqueue.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - clusterqueues - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kueue-x-k8s-io-v1alpha1-cohort - failurePolicy: Fail - name: vcohort.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1alpha1 - operations: - - CREATE - - UPDATE - resources: - - cohorts - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kueue-x-k8s-io-v1beta1-resourceflavor - failurePolicy: Fail - name: vresourceflavor.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - resourceflavors - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kueue-x-k8s-io-v1beta1-workload - failurePolicy: Fail - name: vworkload.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - workloads - - workloads/status - sideEffects: None diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.10.1.yaml b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.10.1.yaml deleted file mode 100644 index 6cd24729b8..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.10.1.yaml +++ /dev/null @@ -1,13175 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Namespace -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-system ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: admissionchecks.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: AdmissionCheck - listKind: AdmissionCheckList - plural: admissionchecks - singular: admissioncheck - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: AdmissionCheck is the Schema for the admissionchecks API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: AdmissionCheckSpec defines the desired state of AdmissionCheck - properties: - controllerName: - description: |- - controllerName identifies the controller that processes the AdmissionCheck, - not necessarily a Kubernetes Pod or Deployment name. Cannot be empty. - type: string - x-kubernetes-validations: - - message: field is immutable - rule: self == oldSelf - parameters: - description: |- - Parameters identifies a configuration with additional parameters for the - check. - properties: - apiGroup: - description: ApiGroup is the group for the resource being referenced. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: Kind is the type of the resource being referenced. - maxLength: 63 - pattern: ^(?i)[a-z]([-a-z0-9]*[a-z0-9])?$ - type: string - name: - description: Name is the name of the resource being referenced. - maxLength: 63 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - apiGroup - - kind - - name - type: object - retryDelayMinutes: - default: 15 - description: |- - RetryDelayMinutes specifies how long to keep the workload suspended after - a failed check (after it transitioned to False). When the delay period has passed, the check - state goes to "Unknown". The default is 15 min. - Deprecated: retryDelayMinutes has already been deprecated since v0.8 and will be removed in v1beta2. - format: int64 - type: integer - required: - - controllerName - type: object - status: - description: AdmissionCheckStatus defines the observed state of AdmissionCheck - properties: - conditions: - description: |- - conditions hold the latest available observations of the AdmissionCheck - current state. - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: clusterqueues.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: ClusterQueue - listKind: ClusterQueueList - plural: clusterqueues - shortNames: - - cq - singular: clusterqueue - scope: Cluster - versions: - - additionalPrinterColumns: - - description: Cohort that this ClusterQueue belongs to - jsonPath: .spec.cohort - name: Cohort - type: string - - description: The queueing strategy used to prioritize workloads - jsonPath: .spec.queueingStrategy - name: Strategy - priority: 1 - type: string - - description: Number of pending workloads - jsonPath: .status.pendingWorkloads - name: Pending Workloads - type: integer - - description: Number of admitted workloads that haven't finished yet - jsonPath: .status.admittedWorkloads - name: Admitted Workloads - priority: 1 - type: integer - name: v1beta1 - schema: - openAPIV3Schema: - description: ClusterQueue is the Schema for the clusterQueue API. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: ClusterQueueSpec defines the desired state of ClusterQueue - properties: - admissionChecks: - description: |- - admissionChecks lists the AdmissionChecks required by this ClusterQueue. - Cannot be used along with AdmissionCheckStrategy. - items: - type: string - type: array - admissionChecksStrategy: - description: |- - admissionCheckStrategy defines a list of strategies to determine which ResourceFlavors require AdmissionChecks. - This property cannot be used in conjunction with the 'admissionChecks' property. - properties: - admissionChecks: - description: admissionChecks is a list of strategies for AdmissionChecks - items: - description: AdmissionCheckStrategyRule defines rules for a - single AdmissionCheck - properties: - name: - description: name is an AdmissionCheck's name. - type: string - onFlavors: - description: |- - onFlavors is a list of ResourceFlavors' names that this AdmissionCheck should run for. - If empty, the AdmissionCheck will run for all workloads submitted to the ClusterQueue. - items: - description: ResourceFlavorReference is the name of the - ResourceFlavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - type: array - required: - - name - type: object - type: array - type: object - cohort: - description: |- - cohort that this ClusterQueue belongs to. CQs that belong to the - same cohort can borrow unused resources from each other. - - A CQ can be a member of a single borrowing cohort. A workload submitted - to a queue referencing this CQ can borrow quota from any CQ in the cohort. - Only quota for the [resource, flavor] pairs listed in the CQ can be - borrowed. - If empty, this ClusterQueue cannot borrow from any other ClusterQueue and - vice versa. - - A cohort is a name that links CQs together, but it doesn't reference any - object. - - Validation of a cohort name is equivalent to that of object names: - subdomain in DNS (RFC 1123). - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - fairSharing: - description: |- - fairSharing defines the properties of the ClusterQueue when participating in fair sharing. - The values are only relevant if fair sharing is enabled in the Kueue configuration. - properties: - weight: - anyOf: - - type: integer - - type: string - default: 1 - description: |- - weight gives a comparative advantage to this ClusterQueue when competing for unused - resources in the cohort against other ClusterQueues. - The share of a ClusterQueue is based on the dominant resource usage above nominal - quotas for each resource, divided by the weight. - Admission prioritizes scheduling workloads from ClusterQueues with the lowest share - and preempting workloads from the ClusterQueues with the highest share. - A zero weight implies infinite share value, meaning that this ClusterQueue will always - be at disadvantage against other ClusterQueues. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - flavorFungibility: - default: {} - description: |- - flavorFungibility defines whether a workload should try the next flavor - before borrowing or preempting in the flavor being evaluated. - properties: - whenCanBorrow: - default: Borrow - description: |- - whenCanBorrow determines whether a workload should try the next flavor - before borrowing in current flavor. The possible values are: - - - `Borrow` (default): allocate in current flavor if borrowing - is possible. - - `TryNextFlavor`: try next flavor even if the current - flavor has enough resources to borrow. - enum: - - Borrow - - TryNextFlavor - type: string - whenCanPreempt: - default: TryNextFlavor - description: |- - whenCanPreempt determines whether a workload should try the next flavor - before borrowing in current flavor. The possible values are: - - - `Preempt`: allocate in current flavor if it's possible to preempt some workloads. - - `TryNextFlavor` (default): try next flavor even if there are enough - candidates for preemption in the current flavor. - enum: - - Preempt - - TryNextFlavor - type: string - type: object - namespaceSelector: - description: |- - namespaceSelector defines which namespaces are allowed to submit workloads to - this clusterQueue. Beyond this basic support for policy, a policy agent like - Gatekeeper should be used to enforce more advanced policies. - Defaults to null which is a nothing selector (no namespaces eligible). - If set to an empty selector `{}`, then all namespaces are eligible. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - preemption: - default: {} - description: |- - preemption describes policies to preempt Workloads from this ClusterQueue - or the ClusterQueue's cohort. - - Preemption can happen in two scenarios: - - - When a Workload fits within the nominal quota of the ClusterQueue, but - the quota is currently borrowed by other ClusterQueues in the cohort. - Preempting Workloads in other ClusterQueues allows this ClusterQueue to - reclaim its nominal quota. - - When a Workload doesn't fit within the nominal quota of the ClusterQueue - and there are admitted Workloads in the ClusterQueue with lower priority. - - The preemption algorithm tries to find a minimal set of Workloads to - preempt to accomomdate the pending Workload, preempting Workloads with - lower priority first. - properties: - borrowWithinCohort: - default: {} - description: |- - borrowWithinCohort provides configuration to allow preemption within - cohort while borrowing. - properties: - maxPriorityThreshold: - description: |- - maxPriorityThreshold allows to restrict the set of workloads which - might be preempted by a borrowing workload, to only workloads with - priority less than or equal to the specified threshold priority. - When the threshold is not specified, then any workload satisfying the - policy can be preempted by the borrowing workload. - format: int32 - type: integer - policy: - default: Never - description: |- - policy determines the policy for preemption to reclaim quota within cohort while borrowing. - Possible values are: - - `Never` (default): do not allow for preemption, in other - ClusterQueues within the cohort, for a borrowing workload. - - `LowerPriority`: allow preemption, in other ClusterQueues - within the cohort, for a borrowing workload, but only if - the preempted workloads are of lower priority. - enum: - - Never - - LowerPriority - type: string - type: object - reclaimWithinCohort: - default: Never - description: |- - reclaimWithinCohort determines whether a pending Workload can preempt - Workloads from other ClusterQueues in the cohort that are using more than - their nominal quota. The possible values are: - - - `Never` (default): do not preempt Workloads in the cohort. - - `LowerPriority`: **Classic Preemption** if the pending Workload - fits within the nominal quota of its ClusterQueue, only preempt - Workloads in the cohort that have lower priority than the pending - Workload. **Fair Sharing** only preempt Workloads in the cohort that - have lower priority than the pending Workload and that satisfy the - fair sharing preemptionStategies. - - `Any`: **Classic Preemption** if the pending Workload fits within - the nominal quota of its ClusterQueue, preempt any Workload in the - cohort, irrespective of priority. **Fair Sharing** preempt Workloads - in the cohort that satisfy the fair sharing preemptionStrategies. - enum: - - Never - - LowerPriority - - Any - type: string - withinClusterQueue: - default: Never - description: |- - withinClusterQueue determines whether a pending Workload that doesn't fit - within the nominal quota for its ClusterQueue, can preempt active Workloads in - the ClusterQueue. The possible values are: - - - `Never` (default): do not preempt Workloads in the ClusterQueue. - - `LowerPriority`: only preempt Workloads in the ClusterQueue that have - lower priority than the pending Workload. - - `LowerOrNewerEqualPriority`: only preempt Workloads in the ClusterQueue that - either have a lower priority than the pending workload or equal priority - and are newer than the pending workload. - enum: - - Never - - LowerPriority - - LowerOrNewerEqualPriority - type: string - type: object - x-kubernetes-validations: - - message: reclaimWithinCohort=Never and borrowWithinCohort.Policy!=Never - rule: '!(self.reclaimWithinCohort == ''Never'' && has(self.borrowWithinCohort) - && self.borrowWithinCohort.policy != ''Never'')' - queueingStrategy: - default: BestEffortFIFO - description: |- - QueueingStrategy indicates the queueing strategy of the workloads - across the queues in this ClusterQueue. - Current Supported Strategies: - - - StrictFIFO: workloads are ordered strictly by creation time. - Older workloads that can't be admitted will block admitting newer - workloads even if they fit available quota. - - BestEffortFIFO: workloads are ordered by creation time, - however older workloads that can't be admitted will not block - admitting newer workloads that fit existing quota. - enum: - - StrictFIFO - - BestEffortFIFO - type: string - resourceGroups: - description: |- - resourceGroups describes groups of resources. - Each resource group defines the list of resources and a list of flavors - that provide quotas for these resources. - Each resource and each flavor can only form part of one resource group. - resourceGroups can be up to 16. - items: - properties: - coveredResources: - description: |- - coveredResources is the list of resources covered by the flavors in this - group. - Examples: cpu, memory, vendor.com/gpu. - The list cannot be empty and it can contain up to 16 resources. - items: - description: ResourceName is the name identifying various - resources in a ResourceList. - type: string - maxItems: 16 - minItems: 1 - type: array - flavors: - description: |- - flavors is the list of flavors that provide the resources of this group. - Typically, different flavors represent different hardware models - (e.g., gpu models, cpu architectures) or pricing models (on-demand vs spot - cpus). - Each flavor MUST list all the resources listed for this group in the same - order as the .resources field. - The list cannot be empty and it can contain up to 16 flavors. - items: - properties: - name: - description: |- - name of this flavor. The name should match the .metadata.name of a - ResourceFlavor. If a matching ResourceFlavor does not exist, the - ClusterQueue will have an Active condition set to False. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: |- - resources is the list of quotas for this flavor per resource. - There could be up to 16 resources. - items: - properties: - borrowingLimit: - anyOf: - - type: integer - - type: string - description: |- - borrowingLimit is the maximum amount of quota for the [flavor, resource] - combination that this ClusterQueue is allowed to borrow from the unused - quota of other ClusterQueues in the same cohort. - In total, at a given time, Workloads in a ClusterQueue can consume a - quantity of quota equal to nominalQuota+borrowingLimit, assuming the other - ClusterQueues in the cohort have enough unused quota. - If null, it means that there is no borrowing limit. - If not null, it must be non-negative. - borrowingLimit must be null if spec.cohort is empty. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - lendingLimit: - anyOf: - - type: integer - - type: string - description: |- - lendingLimit is the maximum amount of unused quota for the [flavor, resource] - combination that this ClusterQueue can lend to other ClusterQueues in the same cohort. - In total, at a given time, ClusterQueue reserves for its exclusive use - a quantity of quota equals to nominalQuota - lendingLimit. - If null, it means that there is no lending limit, meaning that - all the nominalQuota can be borrowed by other clusterQueues in the cohort. - If not null, it must be non-negative. - lendingLimit must be null if spec.cohort is empty. - This field is in beta stage and is enabled by default. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - name: - description: name of this resource. - type: string - nominalQuota: - anyOf: - - type: integer - - type: string - description: |- - nominalQuota is the quantity of this resource that is available for - Workloads admitted by this ClusterQueue at a point in time. - The nominalQuota must be non-negative. - nominalQuota should represent the resources in the cluster available for - running jobs (after discounting resources consumed by system components - and pods not managed by kueue). In an autoscaled cluster, nominalQuota - should account for resources that can be provided by a component such as - Kubernetes cluster-autoscaler. - - If the ClusterQueue belongs to a cohort, the sum of the quotas for each - (flavor, resource) combination defines the maximum quantity that can be - allocated by a ClusterQueue in the cohort. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - - nominalQuota - type: object - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - coveredResources - - flavors - type: object - x-kubernetes-validations: - - message: flavors must have the same number of resources as the - coveredResources - rule: self.flavors.all(x, size(x.resources) == size(self.coveredResources)) - maxItems: 16 - type: array - x-kubernetes-list-type: atomic - stopPolicy: - default: None - description: |- - stopPolicy - if set to a value different from None, the ClusterQueue is considered Inactive, no new reservation being - made. - - Depending on its value, its associated workloads will: - - - None - Workloads are admitted - - HoldAndDrain - Admitted workloads are evicted and Reserving workloads will cancel the reservation. - - Hold - Admitted workloads will run to completion and Reserving workloads will cancel the reservation. - enum: - - None - - Hold - - HoldAndDrain - type: string - type: object - x-kubernetes-validations: - - message: borrowingLimit must be nil when cohort is empty - rule: '!has(self.cohort) && has(self.resourceGroups) ? self.resourceGroups.all(rg, - rg.flavors.all(f, f.resources.all(r, !has(r.borrowingLimit)))) : true' - status: - description: ClusterQueueStatus defines the observed state of ClusterQueue - properties: - admittedWorkloads: - description: |- - admittedWorkloads is the number of workloads currently admitted to this - clusterQueue and haven't finished yet. - format: int32 - type: integer - conditions: - description: |- - conditions hold the latest available observations of the ClusterQueue - current state. - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - fairSharing: - description: FairSharing contains the information about the current - status of fair sharing. - properties: - weightedShare: - description: |- - WeightedShare represent the maximum of the ratios of usage above nominal - quota to the lendable resources in the cohort, among all the resources - provided by the ClusterQueue, and divided by the weight. - If zero, it means that the usage of the ClusterQueue is below the nominal quota. - If the ClusterQueue has a weight of zero, this will return 9223372036854775807, - the maximum possible share value. - format: int64 - type: integer - required: - - weightedShare - type: object - flavorsReservation: - description: |- - flavorsReservation are the reserved quotas, by flavor, currently in use by the - workloads assigned to this ClusterQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: resources lists the quota usage for the resources - in this flavor. - items: - properties: - borrowed: - anyOf: - - type: integer - - type: string - description: |- - Borrowed is quantity of quota that is borrowed from the cohort. In other - words, it's the used quota that is over the nominalQuota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - name: - description: name of the resource - type: string - total: - anyOf: - - type: integer - - type: string - description: |- - total is the total quantity of used quota, including the amount borrowed - from the cohort. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - flavorsUsage: - description: |- - flavorsUsage are the used quotas, by flavor, currently in use by the - workloads admitted in this ClusterQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: resources lists the quota usage for the resources - in this flavor. - items: - properties: - borrowed: - anyOf: - - type: integer - - type: string - description: |- - Borrowed is quantity of quota that is borrowed from the cohort. In other - words, it's the used quota that is over the nominalQuota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - name: - description: name of the resource - type: string - total: - anyOf: - - type: integer - - type: string - description: |- - total is the total quantity of used quota, including the amount borrowed - from the cohort. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - pendingWorkloads: - description: |- - pendingWorkloads is the number of workloads currently waiting to be - admitted to this clusterQueue. - format: int32 - type: integer - pendingWorkloadsStatus: - description: |- - PendingWorkloadsStatus contains the information exposed about the current - status of the pending workloads in the cluster queue. - Deprecated: This field will be removed on v1beta2, use VisibilityOnDemand - (https://kueue.sigs.k8s.io/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand/) - instead. - properties: - clusterQueuePendingWorkload: - description: Head contains the list of top pending workloads. - items: - description: |- - ClusterQueuePendingWorkload contains the information identifying a pending workload - in the cluster queue. - properties: - name: - description: Name indicates the name of the pending workload. - type: string - namespace: - description: Namespace indicates the name of the pending - workload. - type: string - required: - - name - - namespace - type: object - type: array - x-kubernetes-list-type: atomic - lastChangeTime: - description: LastChangeTime indicates the time of the last change - of the structure. - format: date-time - type: string - required: - - lastChangeTime - type: object - reservingWorkloads: - description: |- - reservingWorkloads is the number of workloads currently reserving quota in this - clusterQueue. - format: int32 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: cohorts.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: Cohort - listKind: CohortList - plural: cohorts - singular: cohort - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: |- - Cohort is the Schema for the cohorts API. Using Hierarchical - Cohorts (any Cohort which has a parent) with Fair Sharing - results in undefined behavior in 0.9 - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: CohortSpec defines the desired state of Cohort - properties: - parent: - description: |- - Parent references the name of the Cohort's parent, if - any. It satisfies one of three cases: - 1) Unset. This Cohort is the root of its Cohort tree. - 2) References a non-existent Cohort. We use default Cohort (no borrowing/lending limits). - 3) References an existent Cohort. - - If a cycle is created, we disable all members of the - Cohort, including ClusterQueues, until the cycle is - removed. We prevent further admission while the cycle - exists. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resourceGroups: - description: |- - ResourceGroups describes groupings of Resources and - Flavors. Each ResourceGroup defines a list of Resources - and a list of Flavors which provide quotas for these - Resources. Each Resource and each Flavor may only form part - of one ResourceGroup. There may be up to 16 ResourceGroups - within a Cohort. - - BorrowingLimit limits how much members of this Cohort - subtree can borrow from the parent subtree. - - LendingLimit limits how much members of this Cohort subtree - can lend to the parent subtree. - - Borrowing and Lending limits must only be set when the - Cohort has a parent. Otherwise, the Cohort create/update - will be rejected by the webhook. - items: - properties: - coveredResources: - description: |- - coveredResources is the list of resources covered by the flavors in this - group. - Examples: cpu, memory, vendor.com/gpu. - The list cannot be empty and it can contain up to 16 resources. - items: - description: ResourceName is the name identifying various - resources in a ResourceList. - type: string - maxItems: 16 - minItems: 1 - type: array - flavors: - description: |- - flavors is the list of flavors that provide the resources of this group. - Typically, different flavors represent different hardware models - (e.g., gpu models, cpu architectures) or pricing models (on-demand vs spot - cpus). - Each flavor MUST list all the resources listed for this group in the same - order as the .resources field. - The list cannot be empty and it can contain up to 16 flavors. - items: - properties: - name: - description: |- - name of this flavor. The name should match the .metadata.name of a - ResourceFlavor. If a matching ResourceFlavor does not exist, the - ClusterQueue will have an Active condition set to False. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: |- - resources is the list of quotas for this flavor per resource. - There could be up to 16 resources. - items: - properties: - borrowingLimit: - anyOf: - - type: integer - - type: string - description: |- - borrowingLimit is the maximum amount of quota for the [flavor, resource] - combination that this ClusterQueue is allowed to borrow from the unused - quota of other ClusterQueues in the same cohort. - In total, at a given time, Workloads in a ClusterQueue can consume a - quantity of quota equal to nominalQuota+borrowingLimit, assuming the other - ClusterQueues in the cohort have enough unused quota. - If null, it means that there is no borrowing limit. - If not null, it must be non-negative. - borrowingLimit must be null if spec.cohort is empty. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - lendingLimit: - anyOf: - - type: integer - - type: string - description: |- - lendingLimit is the maximum amount of unused quota for the [flavor, resource] - combination that this ClusterQueue can lend to other ClusterQueues in the same cohort. - In total, at a given time, ClusterQueue reserves for its exclusive use - a quantity of quota equals to nominalQuota - lendingLimit. - If null, it means that there is no lending limit, meaning that - all the nominalQuota can be borrowed by other clusterQueues in the cohort. - If not null, it must be non-negative. - lendingLimit must be null if spec.cohort is empty. - This field is in beta stage and is enabled by default. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - name: - description: name of this resource. - type: string - nominalQuota: - anyOf: - - type: integer - - type: string - description: |- - nominalQuota is the quantity of this resource that is available for - Workloads admitted by this ClusterQueue at a point in time. - The nominalQuota must be non-negative. - nominalQuota should represent the resources in the cluster available for - running jobs (after discounting resources consumed by system components - and pods not managed by kueue). In an autoscaled cluster, nominalQuota - should account for resources that can be provided by a component such as - Kubernetes cluster-autoscaler. - - If the ClusterQueue belongs to a cohort, the sum of the quotas for each - (flavor, resource) combination defines the maximum quantity that can be - allocated by a ClusterQueue in the cohort. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - - nominalQuota - type: object - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - coveredResources - - flavors - type: object - x-kubernetes-validations: - - message: flavors must have the same number of resources as the - coveredResources - rule: self.flavors.all(x, size(x.resources) == size(self.coveredResources)) - maxItems: 16 - type: array - x-kubernetes-list-type: atomic - type: object - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: localqueues.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: LocalQueue - listKind: LocalQueueList - plural: localqueues - shortNames: - - queue - - queues - - lq - singular: localqueue - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Backing ClusterQueue - jsonPath: .spec.clusterQueue - name: ClusterQueue - type: string - - description: Number of pending workloads - jsonPath: .status.pendingWorkloads - name: Pending Workloads - type: integer - - description: Number of admitted workloads that haven't finished yet. - jsonPath: .status.admittedWorkloads - name: Admitted Workloads - type: integer - name: v1beta1 - schema: - openAPIV3Schema: - description: LocalQueue is the Schema for the localQueues API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: LocalQueueSpec defines the desired state of LocalQueue - properties: - clusterQueue: - description: clusterQueue is a reference to a clusterQueue that backs - this localQueue. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - x-kubernetes-validations: - - message: field is immutable - rule: self == oldSelf - stopPolicy: - default: None - description: |- - stopPolicy - if set to a value different from None, the LocalQueue is considered Inactive, - no new reservation being made. - - Depending on its value, its associated workloads will: - - - None - Workloads are admitted - - HoldAndDrain - Admitted workloads are evicted and Reserving workloads will cancel the reservation. - - Hold - Admitted workloads will run to completion and Reserving workloads will cancel the reservation. - enum: - - None - - Hold - - HoldAndDrain - type: string - type: object - status: - description: LocalQueueStatus defines the observed state of LocalQueue - properties: - admittedWorkloads: - description: |- - admittedWorkloads is the number of workloads in this LocalQueue - admitted to a ClusterQueue and that haven't finished yet. - format: int32 - type: integer - conditions: - description: |- - Conditions hold the latest available observations of the LocalQueue - current state. - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - flavorUsage: - description: |- - flavorsUsage are the used quotas, by flavor currently in use by the - workloads assigned to this LocalQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: resources lists the quota usage for the resources - in this flavor. - items: - properties: - name: - description: name of the resource. - type: string - total: - anyOf: - - type: integer - - type: string - description: total is the total quantity of used quota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - flavors: - description: flavors lists all currently available ResourceFlavors - in specified ClusterQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - nodeLabels: - additionalProperties: - type: string - description: |- - nodeLabels are labels that associate the ResourceFlavor with Nodes that - have the same labels. - maxProperties: 8 - type: object - x-kubernetes-map-type: atomic - nodeTaints: - description: |- - nodeTaints are taints that the nodes associated with this ResourceFlavor - have. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Required. The taint key to be applied to - a node. - type: string - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint - key. - type: string - required: - - effect - - key - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - resources: - description: resources used in the flavor. - items: - description: ResourceName is the name identifying various - resources in a ResourceList. - type: string - maxItems: 16 - type: array - x-kubernetes-list-type: set - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - flavorsReservation: - description: |- - flavorsReservation are the reserved quotas, by flavor currently in use by the - workloads assigned to this LocalQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: resources lists the quota usage for the resources - in this flavor. - items: - properties: - name: - description: name of the resource. - type: string - total: - anyOf: - - type: integer - - type: string - description: total is the total quantity of used quota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - pendingWorkloads: - description: PendingWorkloads is the number of Workloads in the LocalQueue - not yet admitted to a ClusterQueue - format: int32 - type: integer - reservingWorkloads: - description: |- - reservingWorkloads is the number of workloads in this LocalQueue - reserving quota in a ClusterQueue and that haven't finished yet. - format: int32 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: multikueueclusters.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: MultiKueueCluster - listKind: MultiKueueClusterList - plural: multikueueclusters - singular: multikueuecluster - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: MultiKueueCluster is the Schema for the multikueue API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - properties: - kubeConfig: - description: Information how to connect to the cluster. - properties: - location: - description: |- - Location of the KubeConfig. - - If LocationType is Secret then Location is the name of the secret inside the namespace in - which the kueue controller manager is running. The config should be stored in the "kubeconfig" key. - type: string - locationType: - default: Secret - description: Type of the KubeConfig location. - enum: - - Secret - - Path - type: string - required: - - location - - locationType - type: object - required: - - kubeConfig - type: object - status: - properties: - conditions: - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: multikueueconfigs.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: MultiKueueConfig - listKind: MultiKueueConfigList - plural: multikueueconfigs - singular: multikueueconfig - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: MultiKueueConfig is the Schema for the multikueue API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: MultiKueueConfigSpec defines the desired state of MultiKueueConfig - properties: - clusters: - description: List of MultiKueueClusters names where the workloads - from the ClusterQueue should be distributed. - items: - type: string - maxItems: 10 - minItems: 1 - type: array - x-kubernetes-list-type: set - required: - - clusters - type: object - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: provisioningrequestconfigs.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: ProvisioningRequestConfig - listKind: ProvisioningRequestConfigList - plural: provisioningrequestconfigs - singular: provisioningrequestconfig - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: ProvisioningRequestConfig is the Schema for the provisioningrequestconfig - API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: ProvisioningRequestConfigSpec defines the desired state of - ProvisioningRequestConfig - properties: - managedResources: - description: |- - managedResources contains the list of resources managed by the autoscaling. - - If empty, all resources are considered managed. - - If not empty, the ProvisioningRequest will contain only the podsets that are - requesting at least one of them. - - If none of the workloads podsets is requesting at least a managed resource, - the workload is considered ready. - items: - description: ResourceName is the name identifying various resources - in a ResourceList. - type: string - maxItems: 100 - type: array - x-kubernetes-list-type: set - parameters: - additionalProperties: - description: Parameter is limited to 255 characters. - maxLength: 255 - type: string - description: Parameters contains all other parameters classes may - require. - maxProperties: 100 - type: object - provisioningClassName: - description: |- - ProvisioningClassName describes the different modes of provisioning the resources. - Check autoscaling.x-k8s.io ProvisioningRequestSpec.ProvisioningClassName for details. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - retryStrategy: - default: - backoffBaseSeconds: 60 - backoffLimitCount: 3 - backoffMaxSeconds: 1800 - description: |- - retryStrategy defines strategy for retrying ProvisioningRequest. - If null, then the default configuration is applied with the following parameter values: - backoffLimitCount: 3 - backoffBaseSeconds: 60 - 1 min - backoffMaxSeconds: 1800 - 30 mins - - To switch off retry mechanism - set retryStrategy.backoffLimitCount to 0. - properties: - backoffBaseSeconds: - default: 60 - description: |- - BackoffBaseSeconds defines the base for the exponential backoff for - re-queuing an evicted workload. - - Defaults to 60. - format: int32 - type: integer - backoffLimitCount: - default: 3 - description: |- - BackoffLimitCount defines the maximum number of re-queuing retries. - Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). - - Every backoff duration is about "b*2^(n-1)+Rand" where: - - "b" represents the base set by "BackoffBaseSeconds" parameter, - - "n" represents the "workloadStatus.requeueState.count", - - "Rand" represents the random jitter. - During this time, the workload is taken as an inadmissible and - other workloads will have a chance to be admitted. - By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...). - - Defaults to 3. - format: int32 - type: integer - backoffMaxSeconds: - default: 1800 - description: |- - BackoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload. - - Defaults to 1800. - format: int32 - type: integer - type: object - required: - - provisioningClassName - type: object - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: resourceflavors.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: ResourceFlavor - listKind: ResourceFlavorList - plural: resourceflavors - shortNames: - - flavor - - flavors - - rf - singular: resourceflavor - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: ResourceFlavor is the Schema for the resourceflavors API. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: ResourceFlavorSpec defines the desired state of the ResourceFlavor - properties: - nodeLabels: - additionalProperties: - type: string - description: |- - nodeLabels are labels that associate the ResourceFlavor with Nodes that - have the same labels. - When a Workload is admitted, its podsets can only get assigned - ResourceFlavors whose nodeLabels match the nodeSelector and nodeAffinity - fields. - Once a ResourceFlavor is assigned to a podSet, the ResourceFlavor's - nodeLabels should be injected into the pods of the Workload by the - controller that integrates with the Workload object. - - nodeLabels can be up to 8 elements. - maxProperties: 8 - type: object - x-kubernetes-map-type: atomic - nodeTaints: - description: |- - nodeTaints are taints that the nodes associated with this ResourceFlavor - have. - Workloads' podsets must have tolerations for these nodeTaints in order to - get assigned this ResourceFlavor during admission. - - An example of a nodeTaint is - cloud.provider.com/preemptible="true":NoSchedule - - nodeTaints can be up to 8 elements. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Required. The taint key to be applied to a node. - type: string - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - required: - - effect - - key - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - x-kubernetes-validations: - - message: 'supported taint effect values: ''NoSchedule'', ''PreferNoSchedule'', - ''NoExecute''' - rule: self.all(x, x.effect in ['NoSchedule', 'PreferNoSchedule', - 'NoExecute']) - tolerations: - description: |- - tolerations are extra tolerations that will be added to the pods admitted in - the quota associated with this resource flavor. - - An example of a toleration is - cloud.provider.com/preemptible="true":NoSchedule - - tolerations can be up to 8 elements. - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - x-kubernetes-validations: - - message: operator must be Exists when 'key' is empty, which means - 'match all values and all keys' - rule: 'self.all(x, !has(x.key) ? x.operator == ''Exists'' : true)' - - message: effect must be 'NoExecute' when 'tolerationSeconds' is - set - rule: 'self.all(x, has(x.tolerationSeconds) ? x.effect == ''NoExecute'' - : true)' - - message: 'supported toleration values: ''Equal''(default), ''Exists''' - rule: self.all(x, !has(x.operator) || x.operator in ['Equal', 'Exists']) - - message: a value must be empty when 'operator' is 'Exists' - rule: 'self.all(x, has(x.operator) && x.operator == ''Exists'' ? - !has(x.value) : true)' - - message: 'supported taint effect values: ''NoSchedule'', ''PreferNoSchedule'', - ''NoExecute''' - rule: self.all(x, !has(x.effect) || x.effect in ['NoSchedule', 'PreferNoSchedule', - 'NoExecute']) - topologyName: - description: |- - topologyName indicates topology for the TAS ResourceFlavor. - When specified, it enables scraping of the topology information from the - nodes matching to the Resource Flavor node labels. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - type: object - x-kubernetes-validations: - - message: at least one nodeLabel is required when topology is set - rule: '!has(self.topologyName) || self.nodeLabels.size() >= 1' - - message: resourceFlavorSpec are immutable when topologyName is set - rule: '!has(oldSelf.topologyName) || self == oldSelf' - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: topologies.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: Topology - listKind: TopologyList - plural: topologies - singular: topology - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: Topology is the Schema for the topology API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: TopologySpec defines the desired state of Topology - properties: - levels: - description: levels define the levels of topology. - items: - description: TopologyLevel defines the desired state of TopologyLevel - properties: - nodeLabel: - description: |- - nodeLabel indicates the name of the node label for a specific topology - level. - - Examples: - - cloud.provider.com/topology-block - - cloud.provider.com/topology-rack - maxLength: 316 - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - nodeLabel - type: object - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - x-kubernetes-validations: - - message: field is immutable - rule: self == oldSelf - - message: must be unique - rule: size(self.filter(i, size(self.filter(j, j == i)) > 1)) == - 0 - - message: the kubernetes.io/hostname label can only be used at the - lowest level of topology - rule: size(self.filter(i, i.nodeLabel == 'kubernetes.io/hostname')) - == 0 || self[size(self) - 1].nodeLabel == 'kubernetes.io/hostname' - required: - - levels - type: object - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: workloadpriorityclasses.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: WorkloadPriorityClass - listKind: WorkloadPriorityClassList - plural: workloadpriorityclasses - singular: workloadpriorityclass - scope: Cluster - versions: - - additionalPrinterColumns: - - description: Value of workloadPriorityClass's Priority - jsonPath: .value - name: Value - type: integer - name: v1beta1 - schema: - openAPIV3Schema: - description: WorkloadPriorityClass is the Schema for the workloadPriorityClass - API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - description: - description: |- - description is an arbitrary string that usually provides guidelines on - when this workloadPriorityClass should be used. - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - value: - description: |- - value represents the integer value of this workloadPriorityClass. This is the actual priority that workloads - receive when jobs have the name of this class in their workloadPriorityClass label. - Changing the value of workloadPriorityClass doesn't affect the priority of workloads that were already created. - format: int32 - type: integer - required: - - value - type: object - served: true - storage: true - subresources: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - controller-gen.kubebuilder.io/version: v0.16.5 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: workloads.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: Workload - listKind: WorkloadList - plural: workloads - shortNames: - - wl - singular: workload - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Name of the queue this workload was submitted to - jsonPath: .spec.queueName - name: Queue - type: string - - description: Name of the ClusterQueue where the workload is reserving quota - jsonPath: .status.admission.clusterQueue - name: Reserved in - type: string - - description: Admission status - jsonPath: .status.conditions[?(@.type=='Admitted')].status - name: Admitted - type: string - - description: Workload finished - jsonPath: .status.conditions[?(@.type=='Finished')].status - name: Finished - type: string - - description: Time this workload was created - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: Workload is the Schema for the workloads API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: WorkloadSpec defines the desired state of Workload - properties: - active: - default: true - description: |- - Active determines if a workload can be admitted into a queue. - Changing active from true to false will evict any running workloads. - Possible values are: - - - false: indicates that a workload should never be admitted and evicts running workloads - - true: indicates that a workload can be evaluated for admission into it's respective queue. - - Defaults to true - type: boolean - maximumExecutionTimeSeconds: - description: |- - maximumExecutionTimeSeconds if provided, determines the maximum time, in seconds, - the workload can be admitted before it's automatically deactivated. - - If unspecified, no execution time limit is enforced on the Workload. - format: int32 - minimum: 1 - type: integer - podSets: - description: |- - podSets is a list of sets of homogeneous pods, each described by a Pod spec - and a count. - There must be at least one element and at most 8. - podSets cannot be changed. - items: - properties: - count: - default: 1 - description: count is the number of pods for the spec. - format: int32 - minimum: 0 - type: integer - minCount: - description: |- - minCount is the minimum number of pods for the spec acceptable - if the workload supports partial admission. - - If not provided, partial admission for the current PodSet is not - enabled. - - Only one podSet within the workload can use this. - - This is an alpha field and requires enabling PartialAdmission feature gate. - format: int32 - minimum: 1 - type: integer - name: - default: main - description: name is the PodSet name. - maxLength: 63 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - template: - description: |- - template is the Pod template. - - The only allowed fields in template.metadata are labels and annotations. - - If requests are omitted for a container or initContainer, - they default to the limits if they are explicitly specified for the - container or initContainer. - - During admission, the rules in nodeSelector and - nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution that match - the keys in the nodeLabels from the ResourceFlavors considered for this - Workload are used to filter the ResourceFlavors that can be assigned to - this podSet. - properties: - metadata: - description: |- - Standard object's metadata. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - Specification of the desired behavior of the pod. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - activeDeadlineSeconds: - description: |- - Optional duration in seconds the pod may be active on the node relative to - StartTime before the system will actively try to mark it failed and kill associated containers. - Value must be a positive integer. - format: int64 - type: integer - affinity: - description: If specified, the pod's scheduling constraints - properties: - nodeAffinity: - description: Describes node affinity scheduling - rules for the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node matches the corresponding matchExpressions; the - node(s) with the highest sum are the most preferred. - items: - description: |- - An empty preferred scheduling term matches all objects with implicit weight 0 - (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated - with the corresponding weight. - properties: - matchExpressions: - description: A list of node selector - requirements by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector - requirements by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - description: Weight associated with matching - the corresponding nodeSelectorTerm, - in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector - terms. The terms are ORed. - items: - description: |- - A null or empty node selector term matches no objects. The requirements of - them are ANDed. - The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector - requirements by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector - requirements by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - description: Describes pod affinity scheduling rules - (e.g. co-locate this pod in the same node, zone, - etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched - WeightedPodAffinityTerm fields are added - per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity - term, associated with the corresponding - weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a - list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a - list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling - rules (e.g. avoid putting this pod in the same - node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the anti-affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched - WeightedPodAffinityTerm fields are added - per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity - term, associated with the corresponding - weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the anti-affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the anti-affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a - list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a - list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - automountServiceAccountToken: - description: AutomountServiceAccountToken indicates - whether a service account token should be automatically - mounted. - type: boolean - containers: - description: |- - List of containers belonging to the pod. - Containers cannot currently be added or removed. - There must be at least one container in a Pod. - Cannot be updated. - items: - description: A single application container that you - want to run within a pod. - properties: - args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment - variable present in a Container. - properties: - name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment - variable's value. Cannot be used if value - is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret - in the pod's namespace - properties: - key: - description: The key of the secret - to select from. Must be a valid - secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source - of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend - to each key in the ConfigMap. Must be - a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action - to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration - that the container should sleep before - being terminated. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action - to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration - that the container should sleep before - being terminated. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action - involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. - type: string - ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network - port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external - port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action - involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents - resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. - type: string - securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label - that applies to the container. - type: string - role: - description: Role is a SELinux role label - that applies to the container. - type: string - type: - description: Type is a SELinux type label - that applies to the container. - type: string - user: - description: User is a SELinux user label - that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is - the name of the GMSA credential spec - to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action - involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block - devices to be used by the container. - items: - description: volumeDevice describes a mapping - of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside - of the container that the device will - be mapped to. - type: string - name: - description: name must match the name of - a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. - items: - description: VolumeMount describes a mounting - of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of - a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - dnsConfig: - description: |- - Specifies the DNS parameters of a pod. - Parameters specified here will be merged to the generated DNS - configuration based on DNSPolicy. - properties: - nameservers: - description: |- - A list of DNS name server IP addresses. - This will be appended to the base nameservers generated from DNSPolicy. - Duplicated nameservers will be removed. - items: - type: string - type: array - x-kubernetes-list-type: atomic - options: - description: |- - A list of DNS resolver options. - This will be merged with the base options generated from DNSPolicy. - Duplicated entries will be removed. Resolution options given in Options - will override those that appear in the base DNSPolicy. - items: - description: PodDNSConfigOption defines DNS resolver - options of a pod. - properties: - name: - description: Required. - type: string - value: - type: string - type: object - type: array - x-kubernetes-list-type: atomic - searches: - description: |- - A list of DNS search domains for host-name lookup. - This will be appended to the base search paths generated from DNSPolicy. - Duplicated search paths will be removed. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - dnsPolicy: - description: |- - Set DNS policy for the pod. - Defaults to "ClusterFirst". - Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. - DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. - To have DNS options set along with hostNetwork, you have to specify DNS policy - explicitly to 'ClusterFirstWithHostNet'. - type: string - enableServiceLinks: - description: |- - EnableServiceLinks indicates whether information about services should be injected into pod's - environment variables, matching the syntax of Docker links. - Optional: Defaults to true. - type: boolean - ephemeralContainers: - description: |- - List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing - pod to perform user-initiated actions such as debugging. This list cannot be specified when - creating a pod, and it cannot be modified by updating the pod spec. In order to add an - ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. - items: - description: |- - An EphemeralContainer is a temporary container that you may add to an existing Pod for - user-initiated activities such as debugging. Ephemeral containers have no resource or - scheduling guarantees, and they will not be restarted when they exit or when a Pod is - removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the - Pod to exceed its resource allocation. - - To add an ephemeral container, use the ephemeralcontainers subresource of an existing - Pod. Ephemeral containers may not be removed or restarted. - properties: - args: - description: |- - Arguments to the entrypoint. - The image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment - variable present in a Container. - properties: - name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment - variable's value. Cannot be used if value - is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret - in the pod's namespace - properties: - key: - description: The key of the secret - to select from. Must be a valid - secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source - of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend - to each key in the ConfigMap. Must be - a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: Lifecycle is not allowed for ephemeral - containers. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action - to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration - that the container should sleep before - being terminated. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action - to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration - that the container should sleep before - being terminated. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: Probes are not allowed for ephemeral - containers. - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action - involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the ephemeral container specified as a DNS_LABEL. - This name must be unique among all containers, init containers and ephemeral containers. - type: string - ports: - description: Ports are not allowed for ephemeral - containers. - items: - description: ContainerPort represents a network - port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external - port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: Probes are not allowed for ephemeral - containers. - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action - involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents - resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources - already allocated to the pod. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - Restart policy for the container to manage the restart behavior of each - container within a pod. - This may only be set for init containers. You cannot set this field on - ephemeral containers. - type: string - securityContext: - description: |- - Optional: SecurityContext defines the security options the ephemeral container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label - that applies to the container. - type: string - role: - description: Role is a SELinux role label - that applies to the container. - type: string - type: - description: Type is a SELinux type label - that applies to the container. - type: string - user: - description: User is a SELinux user label - that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is - the name of the GMSA credential spec - to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: Probes are not allowed for ephemeral - containers. - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action - involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - targetContainerName: - description: |- - If set, the name of the container from PodSpec that this ephemeral container targets. - The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. - If not set then the ephemeral container uses the namespaces configured in the Pod spec. - - The container runtime must implement support for this feature. If the runtime does not - support namespace targeting then the result of setting this field is undefined. - type: string - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block - devices to be used by the container. - items: - description: volumeDevice describes a mapping - of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside - of the container that the device will - be mapped to. - type: string - name: - description: name must match the name of - a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. - Cannot be updated. - items: - description: VolumeMount describes a mounting - of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of - a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - hostAliases: - description: |- - HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts - file if specified. - items: - description: |- - HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the - pod's hosts file. - properties: - hostnames: - description: Hostnames for the above IP address. - items: - type: string - type: array - x-kubernetes-list-type: atomic - ip: - description: IP address of the host file entry. - type: string - required: - - ip - type: object - type: array - x-kubernetes-list-map-keys: - - ip - x-kubernetes-list-type: map - hostIPC: - description: |- - Use the host's ipc namespace. - Optional: Default to false. - type: boolean - hostNetwork: - description: |- - Host networking requested for this pod. Use the host's network namespace. - If this option is set, the ports that will be used must be specified. - Default to false. - type: boolean - hostPID: - description: |- - Use the host's pid namespace. - Optional: Default to false. - type: boolean - hostUsers: - description: |- - Use the host's user namespace. - Optional: Default to true. - If set to true or not present, the pod will be run in the host user namespace, useful - for when the pod needs a feature only available to the host user namespace, such as - loading a kernel module with CAP_SYS_MODULE. - When set to false, a new userns is created for the pod. Setting false is useful for - mitigating container breakout vulnerabilities even allowing users to run their - containers as root without actually having root privileges on the host. - This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. - type: boolean - hostname: - description: |- - Specifies the hostname of the Pod - If not specified, the pod's hostname will be set to a system-defined value. - type: string - imagePullSecrets: - description: |- - ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. - If specified, these secrets will be passed to individual puller implementations for them to use. - More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod - items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - initContainers: - description: |- - List of initialization containers belonging to the pod. - Init containers are executed in order prior to containers being started. If any - init container fails, the pod is considered to have failed and is handled according - to its restartPolicy. The name for an init container or normal container must be - unique among all containers. - Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. - The resourceRequirements of an init container are taken into account during scheduling - by finding the highest request/limit for each resource type, and then using the max of - of that value or the sum of the normal containers. Limits are applied to init containers - in a similar fashion. - Init containers cannot currently be added or removed. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ - items: - description: A single application container that you - want to run within a pod. - properties: - args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment - variable present in a Container. - properties: - name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment - variable's value. Cannot be used if value - is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret - in the pod's namespace - properties: - key: - description: The key of the secret - to select from. Must be a valid - secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source - of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend - to each key in the ConfigMap. Must be - a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action - to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration - that the container should sleep before - being terminated. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies the action - to take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents the duration - that the container should sleep before - being terminated. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for the backward compatibility. There are no validation of this field and - lifecycle hooks will fail in runtime when tcp handler is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action - involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. - type: string - ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network - port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external - port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action - involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents - resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. - type: string - securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label - that applies to the container. - type: string - role: - description: Role is a SELinux role label - that applies to the container. - type: string - type: - description: Type is a SELinux type label - that applies to the container. - type: string - user: - description: User is a SELinux user label - that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is - the name of the GMSA credential spec - to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies the action to - take. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies an action involving - a GRPC port. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies the http request - to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies an action - involving a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block - devices to be used by the container. - items: - description: volumeDevice describes a mapping - of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside - of the container that the device will - be mapped to. - type: string - name: - description: name must match the name of - a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. - items: - description: VolumeMount describes a mounting - of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of - a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - nodeName: - description: |- - NodeName indicates in which node this pod is scheduled. - If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. - Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. - This field should not be used to express a desire for the pod to be scheduled on a specific node. - https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename - type: string - nodeSelector: - additionalProperties: - type: string - description: |- - NodeSelector is a selector which must be true for the pod to fit on a node. - Selector which must match a node's labels for the pod to be scheduled on that node. - More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - type: object - x-kubernetes-map-type: atomic - os: - description: |- - Specifies the OS of the containers in the pod. - Some pod and container fields are restricted if this is set. - - If the OS field is set to linux, the following fields must be unset: - -securityContext.windowsOptions - - If the OS field is set to windows, following fields must be unset: - - spec.hostPID - - spec.hostIPC - - spec.hostUsers - - spec.securityContext.appArmorProfile - - spec.securityContext.seLinuxOptions - - spec.securityContext.seccompProfile - - spec.securityContext.fsGroup - - spec.securityContext.fsGroupChangePolicy - - spec.securityContext.sysctls - - spec.shareProcessNamespace - - spec.securityContext.runAsUser - - spec.securityContext.runAsGroup - - spec.securityContext.supplementalGroups - - spec.securityContext.supplementalGroupsPolicy - - spec.containers[*].securityContext.appArmorProfile - - spec.containers[*].securityContext.seLinuxOptions - - spec.containers[*].securityContext.seccompProfile - - spec.containers[*].securityContext.capabilities - - spec.containers[*].securityContext.readOnlyRootFilesystem - - spec.containers[*].securityContext.privileged - - spec.containers[*].securityContext.allowPrivilegeEscalation - - spec.containers[*].securityContext.procMount - - spec.containers[*].securityContext.runAsUser - - spec.containers[*].securityContext.runAsGroup - properties: - name: - description: |- - Name is the name of the operating system. The currently supported values are linux and windows. - Additional value may be defined in future and can be one of: - https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration - Clients should expect to handle additional values and treat unrecognized values in this field as os: null - type: string - required: - - name - type: object - overhead: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. - This field will be autopopulated at admission time by the RuntimeClass admission controller. If - the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. - The RuntimeClass admission controller will reject Pod create requests which have the overhead already - set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value - defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. - More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md - type: object - preemptionPolicy: - description: |- - PreemptionPolicy is the Policy for preempting pods with lower priority. - One of Never, PreemptLowerPriority. - Defaults to PreemptLowerPriority if unset. - type: string - priority: - description: |- - The priority value. Various system components use this field to find the - priority of the pod. When Priority Admission Controller is enabled, it - prevents users from setting this field. The admission controller populates - this field from PriorityClassName. - The higher the value, the higher the priority. - format: int32 - type: integer - priorityClassName: - description: |- - If specified, indicates the pod's priority. "system-node-critical" and - "system-cluster-critical" are two special keywords which indicate the - highest priorities with the former being the highest priority. Any other - name must be defined by creating a PriorityClass object with that name. - If not specified, the pod priority will be default or zero if there is no - default. - type: string - readinessGates: - description: |- - If specified, all readiness gates will be evaluated for pod readiness. - A pod is ready when all its containers are ready AND - all conditions specified in the readiness gates have status equal to "True" - More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates - items: - description: PodReadinessGate contains the reference - to a pod condition - properties: - conditionType: - description: ConditionType refers to a condition - in the pod's condition list with matching type. - type: string - required: - - conditionType - type: object - type: array - x-kubernetes-list-type: atomic - resourceClaims: - description: |- - ResourceClaims defines which ResourceClaims must be allocated - and reserved before the Pod is allowed to start. The resources - will be made available to those containers which consume them - by name. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. - items: - description: |- - PodResourceClaim references exactly one ResourceClaim, either directly - or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim - for the pod. - - It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. - Containers that need access to the ResourceClaim reference it with this name. - properties: - name: - description: |- - Name uniquely identifies this resource claim inside the pod. - This must be a DNS_LABEL. - type: string - resourceClaimName: - description: |- - ResourceClaimName is the name of a ResourceClaim object in the same - namespace as this pod. - - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. - type: string - resourceClaimTemplateName: - description: |- - ResourceClaimTemplateName is the name of a ResourceClaimTemplate - object in the same namespace as this pod. - - The template will be used to create a new ResourceClaim, which will - be bound to this pod. When this pod is deleted, the ResourceClaim - will also be deleted. The pod name and resource name, along with a - generated component, will be used to form a unique name for the - ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. - - This field is immutable and no changes will be made to the - corresponding ResourceClaim by the control plane after creating the - ResourceClaim. - - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - restartPolicy: - description: |- - Restart policy for all containers within the pod. - One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. - Default to Always. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy - type: string - runtimeClassName: - description: |- - RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used - to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. - If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an - empty definition that uses the default runtime handler. - More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class - type: string - schedulerName: - description: |- - If specified, the pod will be dispatched by specified scheduler. - If not specified, the pod will be dispatched by default scheduler. - type: string - schedulingGates: - description: |- - SchedulingGates is an opaque list of values that if specified will block scheduling the pod. - If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the - scheduler will not attempt to schedule the pod. - - SchedulingGates can only be set at pod creation time, and be removed only afterwards. - items: - description: PodSchedulingGate is associated to a - Pod to guard its scheduling. - properties: - name: - description: |- - Name of the scheduling gate. - Each scheduling gate must have a unique name field. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - securityContext: - description: |- - SecurityContext holds pod-level security attributes and common container settings. - Optional: Defaults to empty. See type description for default values of each field. - properties: - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - fsGroup: - description: |- - A special supplemental group that applies to all containers in a pod. - Some volume types allow the Kubelet to change the ownership of that volume - to be owned by the pod: - - 1. The owning GID will be the FSGroup - 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) - 3. The permission bits are OR'd with rw-rw---- - - If unset, the Kubelet will not modify the ownership and permissions of any volume. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - fsGroupChangePolicy: - description: |- - fsGroupChangePolicy defines behavior of changing ownership and permission of the volume - before being exposed inside Pod. This field will only apply to - volume types which support fsGroup based ownership(and permissions). - It will have no effect on ephemeral volume types such as: secret, configmaps - and emptydir. - Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. - Note that this field cannot be set when spec.os.name is windows. - type: string - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in SecurityContext. If set in - both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that - applies to the container. - type: string - role: - description: Role is a SELinux role label that - applies to the container. - type: string - type: - description: Type is a SELinux type label that - applies to the container. - type: string - user: - description: User is a SELinux user label that - applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - supplementalGroups: - description: |- - A list of groups applied to the first process run in each container, in - addition to the container's primary GID and fsGroup (if specified). If - the SupplementalGroupsPolicy feature is enabled, the - supplementalGroupsPolicy field determines whether these are in addition - to or instead of any group memberships defined in the container image. - If unspecified, no additional groups are added, though group memberships - defined in the container image may still be used, depending on the - supplementalGroupsPolicy field. - Note that this field cannot be set when spec.os.name is windows. - items: - format: int64 - type: integer - type: array - x-kubernetes-list-type: atomic - supplementalGroupsPolicy: - description: |- - Defines how supplemental groups of the first container processes are calculated. - Valid values are "Merge" and "Strict". If not specified, "Merge" is used. - (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled - and the container runtime must implement support for this feature. - Note that this field cannot be set when spec.os.name is windows. - type: string - sysctls: - description: |- - Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported - sysctls (by the container runtime) might fail to launch. - Note that this field cannot be set when spec.os.name is windows. - items: - description: Sysctl defines a kernel parameter - to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options within a container's SecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name - of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - serviceAccount: - description: |- - DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. - Deprecated: Use serviceAccountName instead. - type: string - serviceAccountName: - description: |- - ServiceAccountName is the name of the ServiceAccount to use to run this pod. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ - type: string - setHostnameAsFQDN: - description: |- - If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). - In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). - In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. - If a pod does not have FQDN, this has no effect. - Default to false. - type: boolean - shareProcessNamespace: - description: |- - Share a single process namespace between all of the containers in a pod. - When this is set containers will be able to view and signal processes from other containers - in the same pod, and the first process in each container will not be assigned PID 1. - HostPID and ShareProcessNamespace cannot both be set. - Optional: Default to false. - type: boolean - subdomain: - description: |- - If specified, the fully qualified Pod hostname will be "...svc.". - If not specified, the pod will not have a domainname at all. - type: string - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - If this value is nil, the default grace period will be used instead. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - Defaults to 30 seconds. - format: int64 - type: integer - tolerations: - description: If specified, the pod's tolerations. - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - topologySpreadConstraints: - description: |- - TopologySpreadConstraints describes how a group of pods ought to spread across topology - domains. Scheduler will schedule pods in a way which abides by the constraints. - All topologySpreadConstraints are ANDed. - items: - description: TopologySpreadConstraint specifies how - to spread matching pods among the given topology. - properties: - labelSelector: - description: |- - LabelSelector is used to find matching pods. - Pods that match this label selector are counted to determine the number of pods - in their corresponding topology domain. - properties: - matchExpressions: - description: matchExpressions is a list of - label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select the pods over which - spreading will be calculated. The keys are used to lookup values from the - incoming pod labels, those key-value labels are ANDed with labelSelector - to select the group of existing pods over which spreading will be calculated - for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - MatchLabelKeys cannot be set when LabelSelector isn't set. - Keys that don't exist in the incoming pod labels will - be ignored. A null or empty list means only match against labelSelector. - - This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxSkew: - description: |- - MaxSkew describes the degree to which pods may be unevenly distributed. - When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference - between the number of matching pods in the target topology and the global minimum. - The global minimum is the minimum number of matching pods in an eligible domain - or zero if the number of eligible domains is less than MinDomains. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 2/2/1: - In this case, the global minimum is 1. - | zone1 | zone2 | zone3 | - | P P | P P | P | - - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; - scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) - violate MaxSkew(1). - - if MaxSkew is 2, incoming pod can be scheduled onto any zone. - When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence - to topologies that satisfy it. - It's a required field. Default value is 1 and 0 is not allowed. - format: int32 - type: integer - minDomains: - description: |- - MinDomains indicates a minimum number of eligible domains. - When the number of eligible domains with matching topology keys is less than minDomains, - Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. - And when the number of eligible domains with matching topology keys equals or greater than minDomains, - this value has no effect on scheduling. - As a result, when the number of eligible domains is less than minDomains, - scheduler won't schedule more than maxSkew Pods to those domains. - If value is nil, the constraint behaves as if MinDomains is equal to 1. - Valid values are integers greater than 0. - When value is not nil, WhenUnsatisfiable must be DoNotSchedule. - - For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same - labelSelector spread as 2/2/2: - | zone1 | zone2 | zone3 | - | P P | P P | P P | - The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. - In this situation, new pod with the same labelSelector cannot be scheduled, - because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, - it will violate MaxSkew. - format: int32 - type: integer - nodeAffinityPolicy: - description: |- - NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector - when calculating pod topology spread skew. Options are: - - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. - - If this value is nil, the behavior is equivalent to the Honor policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. - type: string - nodeTaintsPolicy: - description: |- - NodeTaintsPolicy indicates how we will treat node taints when calculating - pod topology spread skew. Options are: - - Honor: nodes without taints, along with tainted nodes for which the incoming pod - has a toleration, are included. - - Ignore: node taints are ignored. All nodes are included. - - If this value is nil, the behavior is equivalent to the Ignore policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. - type: string - topologyKey: - description: |- - TopologyKey is the key of node labels. Nodes that have a label with this key - and identical values are considered to be in the same topology. - We consider each as a "bucket", and try to put balanced number - of pods into each bucket. - We define a domain as a particular instance of a topology. - Also, we define an eligible domain as a domain whose nodes meet the requirements of - nodeAffinityPolicy and nodeTaintsPolicy. - e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. - And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. - It's a required field. - type: string - whenUnsatisfiable: - description: |- - WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy - the spread constraint. - - DoNotSchedule (default) tells the scheduler not to schedule it. - - ScheduleAnyway tells the scheduler to schedule the pod in any location, - but giving higher precedence to topologies that would help reduce the - skew. - A constraint is considered "Unsatisfiable" for an incoming pod - if and only if every possible node assignment for that pod would violate - "MaxSkew" on some topology. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 3/1/1: - | zone1 | zone2 | zone3 | - | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled - to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies - MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler - won't make it *more* imbalanced. - It's a required field. - type: string - required: - - maxSkew - - topologyKey - - whenUnsatisfiable - type: object - type: array - x-kubernetes-list-map-keys: - - topologyKey - - whenUnsatisfiable - x-kubernetes-list-type: map - volumes: - description: |- - List of volumes that can be mounted by containers belonging to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes - items: - description: Volume represents a named volume in a - pod that may be accessed by any container in the - pod. - properties: - awsElasticBlockStore: - description: |- - awsElasticBlockStore represents an AWS Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - format: int32 - type: integer - readOnly: - description: |- - readOnly value true will force the readOnly setting in VolumeMounts. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: boolean - volumeID: - description: |- - volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: string - required: - - volumeID - type: object - azureDisk: - description: azureDisk represents an Azure Data - Disk mount on the host and bind mount to the - pod. - properties: - cachingMode: - description: 'cachingMode is the Host Caching - mode: None, Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data - disk in the blob storage - type: string - diskURI: - description: diskURI is the URI of data disk - in the blob storage - type: string - fsType: - default: ext4 - description: |- - fsType is Filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - kind: - description: 'kind expected values are Shared: - multiple blob disks per storage account Dedicated: - single blob disk per storage account Managed: - azure managed data disk (only in managed - availability set). defaults to shared' - type: string - readOnly: - default: false - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: azureFile represents an Azure File - Service mount on the host and bind mount to - the pod. - properties: - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret - that contains Azure Storage Account Name - and Key - type: string - shareName: - description: shareName is the azure share - Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: cephFS represents a Ceph FS mount - on the host that shares a pod's lifetime - properties: - monitors: - description: |- - monitors is Required: Monitors is a collection of Ceph monitors - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - items: - type: string - type: array - x-kubernetes-list-type: atomic - path: - description: 'path is Optional: Used as the - mounted root, rather than the full Ceph - tree, default is /' - type: string - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: boolean - secretFile: - description: |- - secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - secretRef: - description: |- - secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - user: - description: |- - user is optional: User is the rados user name, default is admin - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - required: - - monitors - type: object - cinder: - description: |- - cinder represents a cinder volume attached and mounted on kubelets host machine. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: boolean - secretRef: - description: |- - secretRef is optional: points to a secret object containing parameters used to connect - to OpenStack. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - volumeID: - description: |- - volumeID used to identify the volume in cinder. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap - that should populate this volume - properties: - defaultMode: - description: |- - defaultMode is optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path - within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional specify whether the - ConfigMap or its keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - csi: - description: csi (Container Storage Interface) - represents ephemeral storage that is handled - by certain external CSI drivers (Beta feature). - properties: - driver: - description: |- - driver is the name of the CSI driver that handles this volume. - Consult with your admin for the correct name as registered in the cluster. - type: string - fsType: - description: |- - fsType to mount. Ex. "ext4", "xfs", "ntfs". - If not provided, the empty value is passed to the associated CSI driver - which will determine the default filesystem to apply. - type: string - nodePublishSecretRef: - description: |- - nodePublishSecretRef is a reference to the secret object containing - sensitive information to pass to the CSI driver to complete the CSI - NodePublishVolume and NodeUnpublishVolume calls. - This field is optional, and may be empty if no secret is required. If the - secret object contains more than one secret, all secret references are passed. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - readOnly: - description: |- - readOnly specifies a read-only configuration for the volume. - Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: |- - volumeAttributes stores driver-specific properties that are passed to the CSI - driver. Consult your driver's documentation for supported values. - type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API - about the pod that should populate this volume - properties: - defaultMode: - description: |- - Optional: mode bits to use on created files by default. Must be a - Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: Items is a list of downward API - volume file - items: - description: DownwardAPIVolumeFile represents - information to create the file containing - the pod field - properties: - fieldRef: - description: 'Required: Selects a field - of the pod: only annotations, labels, - name, namespace and uid are supported.' - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path is the - relative path name of the file to - be created. Must not be absolute or - contain the ''..'' path. Must be utf-8 - encoded. The first item of the relative - path must not start with ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - emptyDir: - description: |- - emptyDir represents a temporary directory that shares a pod's lifetime. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - properties: - medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: |- - ephemeral represents a volume that is handled by a cluster storage driver. - The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, - and deleted when the pod is removed. - - Use this if: - a) the volume is only needed while the pod runs, - b) features of normal volumes like restoring from snapshot or capacity - tracking are needed, - c) the storage driver is specified through a storage class, and - d) the storage driver supports dynamic volume provisioning through - a PersistentVolumeClaim (see EphemeralVolumeSource for more - information on the connection between this volume type - and PersistentVolumeClaim). - - Use PersistentVolumeClaim or one of the vendor-specific - APIs for volumes that persist for longer than the lifecycle - of an individual pod. - - Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to - be used that way - see the documentation of the driver for - more information. - - A pod can use both types of ephemeral volumes and - persistent volumes at the same time. - properties: - volumeClaimTemplate: - description: |- - Will be used to create a stand-alone PVC to provision the volume. - The pod in which this EphemeralVolumeSource is embedded will be the - owner of the PVC, i.e. the PVC will be deleted together with the - pod. The name of the PVC will be `-` where - `` is the name from the `PodSpec.Volumes` array - entry. Pod validation will reject the pod if the concatenated name - is not valid for a PVC (for example, too long). - - An existing PVC with that name that is not owned by the pod - will *not* be used for the pod to avoid using an unrelated - volume by mistake. Starting the pod is then blocked until - the unrelated PVC is removed. If such a pre-created PVC is - meant to be used by the pod, the PVC has to updated with an - owner reference to the pod once the pod exists. Normally - this should not be necessary, but it may be useful when - manually reconstructing a broken cluster. - - This field is read-only and no changes will be made by Kubernetes - to the PVC after it has been created. - - Required, must not be nil. - properties: - metadata: - description: |- - May contain labels and annotations that will be copied into the PVC - when creating it. No other fields are allowed and will be rejected during - validation. - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - The specification for the PersistentVolumeClaim. The entire content is - copied unchanged into the PVC that gets created from this - template. The same fields as in a PersistentVolumeClaim - are also valid here. - properties: - accessModes: - description: |- - accessModes contains the desired access modes the volume should have. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 - items: - type: string - type: array - x-kubernetes-list-type: atomic - dataSource: - description: |- - dataSource field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - If the provisioner or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified data source. - When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, - and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. - If the namespace is specified, then dataSourceRef will not be copied to dataSource. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type - of resource being referenced - type: string - name: - description: Name is the name - of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: |- - dataSourceRef specifies the object from which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a non-empty API group (non - core object) or a PersistentVolumeClaim object. - When this field is specified, volume binding will only succeed if the type of - the specified object matches some installed volume populator or dynamic - provisioner. - This field will replace the functionality of the dataSource field and as such - if both fields are non-empty, they must have the same value. For backwards - compatibility, when namespace isn't specified in dataSourceRef, - both fields (dataSource and dataSourceRef) will be set to the same - value automatically if one of them is empty and the other is non-empty. - When namespace is specified in dataSourceRef, - dataSource isn't set to the same value and must be empty. - There are three important differences between dataSource and dataSourceRef: - * While dataSource only allows two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping them), dataSourceRef - preserves all values, and generates an error if a disallowed value is - specified. - * While dataSource only allows local objects, dataSourceRef allows objects - in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. - (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type - of resource being referenced - type: string - name: - description: Name is the name - of resource being referenced - type: string - namespace: - description: |- - Namespace is the namespace of resource being referenced - Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. - (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: |- - resources represents the minimum resources the volume should have. - If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements - that are lower than previous value but must still be higher than capacity recorded in the - status field of the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - description: selector is a label query - over volumes to consider for binding. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: |- - storageClassName is the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 - type: string - volumeAttributesClassName: - description: |- - volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. - If specified, the CSI driver will create or update the volume with the attributes defined - in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. - If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be - set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource - exists. - More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). - type: string - volumeMode: - description: |- - volumeMode defines what type of volume is required by the claim. - Value of Filesystem is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding - reference to the PersistentVolume - backing this claim. - type: string - type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource - that is attached to a kubelet's host machine - and then exposed to the pod. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - lun: - description: 'lun is Optional: FC target lun - number' - format: int32 - type: integer - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target - worldwide names (WWNs)' - items: - type: string - type: array - x-kubernetes-list-type: atomic - wwids: - description: |- - wwids Optional: FC volume world wide identifiers (wwids) - Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - flexVolume: - description: |- - flexVolume represents a generic volume resource that is - provisioned/attached using an exec based plugin. - properties: - driver: - description: driver is the name of the driver - to use for this volume. - type: string - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field - holds extra command options if any.' - type: object - readOnly: - description: |- - readOnly is Optional: defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef is Optional: secretRef is reference to the secret object containing - sensitive information to pass to the plugin scripts. This may be - empty if no secret object is specified. If the secret object - contains more than one secret, all secrets are passed to the plugin - scripts. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - required: - - driver - type: object - flocker: - description: flocker represents a Flocker volume - attached to a kubelet's host machine. This depends - on the Flocker control service being running - properties: - datasetName: - description: |- - datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker - should be considered as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the - dataset. This is unique identifier of a - Flocker dataset - type: string - type: object - gcePersistentDisk: - description: |- - gcePersistentDisk represents a GCE Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - properties: - fsType: - description: |- - fsType is filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - format: int32 - type: integer - pdName: - description: |- - pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: boolean - required: - - pdName - type: object - gitRepo: - description: |- - gitRepo represents a git repository at a particular revision. - DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an - EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir - into the Pod's container. - properties: - directory: - description: |- - directory is the target directory name. - Must not contain or start with '..'. If '.' is supplied, the volume directory will be the - git repository. Otherwise, if specified, the volume will contain the git repository in - the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for - the specified revision. - type: string - required: - - repository - type: object - glusterfs: - description: |- - glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. - More info: https://examples.k8s.io/volumes/glusterfs/README.md - properties: - endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: string - path: - description: |- - path is the Glusterfs volume path. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: string - readOnly: - description: |- - readOnly here will force the Glusterfs volume to be mounted with read-only permissions. - Defaults to false. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: boolean - required: - - endpoints - - path - type: object - hostPath: - description: |- - hostPath represents a pre-existing file or directory on the host - machine that is directly exposed to the container. This is generally - used for system agents or other privileged things that are allowed - to see the host machine. Most containers will NOT need this. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - properties: - path: - description: |- - path of the directory on the host. - If the path is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - type: - description: |- - type for HostPath Volume - Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - required: - - path - type: object - image: - description: |- - image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. - The volume is resolved at pod startup depending on which PullPolicy value is provided: - - - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - - The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. - A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. - The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. - The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. - The volume will be mounted read-only (ro) and non-executable files (noexec). - Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). - The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. - properties: - pullPolicy: - description: |- - Policy for pulling OCI objects. Possible values are: - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - type: string - reference: - description: |- - Required: Image or artifact reference to be used. - Behaves in the same way as pod.spec.containers[*].image. - Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - type: object - iscsi: - description: |- - iscsi represents an ISCSI Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md - properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether - support iSCSI Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether - support iSCSI Session CHAP authentication - type: boolean - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - type: string - initiatorName: - description: |- - initiatorName is the custom iSCSI Initiator Name. - If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface - : will be created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified - Name. - type: string - iscsiInterface: - default: default - description: |- - iscsiInterface is the interface Name that uses an iSCSI transport. - Defaults to 'default' (tcp). - type: string - lun: - description: lun represents iSCSI Target Lun - number. - format: int32 - type: integer - portals: - description: |- - portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - items: - type: string - type: array - x-kubernetes-list-type: atomic - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret - for iSCSI target and initiator authentication - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - targetPortal: - description: |- - targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - description: |- - name of the volume. - Must be a DNS_LABEL and unique within the pod. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - nfs: - description: |- - nfs represents an NFS mount on the host that shares a pod's lifetime - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - properties: - path: - description: |- - path that is exported by the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - readOnly: - description: |- - readOnly here will force the NFS export to be mounted with read-only permissions. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: boolean - server: - description: |- - server is the hostname or IP address of the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: |- - persistentVolumeClaimVolumeSource represents a reference to a - PersistentVolumeClaim in the same namespace. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - properties: - claimName: - description: |- - claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - type: string - readOnly: - description: |- - readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: photonPersistentDisk represents a - PhotonController persistent disk attached and - mounted on kubelets host machine - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - pdID: - description: pdID is the ID that identifies - Photon Controller persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: portworxVolume represents a portworx - volume attached and mounted on kubelets host - machine - properties: - fsType: - description: |- - fSType represents the filesystem type to mount - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies - a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources - secrets, configmaps, and downward API - properties: - defaultMode: - description: |- - defaultMode are the mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - sources: - description: |- - sources is the list of volume projections. Each entry in this list - handles one source. - items: - description: |- - Projection that may be projected along with other supported volume types. - Exactly one of these fields must be set. - properties: - clusterTrustBundle: - description: |- - ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field - of ClusterTrustBundle objects in an auto-updating file. - - Alpha, gated by the ClusterTrustBundleProjection feature gate. - - ClusterTrustBundle objects can either be selected by name, or by the - combination of signer name and a label selector. - - Kubelet performs aggressive normalization of the PEM contents written - into the pod filesystem. Esoteric PEM features such as inter-block - comments and block headers are stripped. Certificates are deduplicated. - The ordering of certificates within the file is arbitrary, and Kubelet - may change the order over time. - properties: - labelSelector: - description: |- - Select all ClusterTrustBundles that match this label selector. Only has - effect if signerName is set. Mutually-exclusive with name. If unset, - interpreted as "match nothing". If set but empty, interpreted as "match - everything". - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - name: - description: |- - Select a single ClusterTrustBundle by object name. Mutually-exclusive - with signerName and labelSelector. - type: string - optional: - description: |- - If true, don't block pod startup if the referenced ClusterTrustBundle(s) - aren't available. If using name, then the named ClusterTrustBundle is - allowed not to exist. If using signerName, then the combination of - signerName and labelSelector is allowed to match zero - ClusterTrustBundles. - type: boolean - path: - description: Relative path from - the volume root to write the bundle. - type: string - signerName: - description: |- - Select all ClusterTrustBundles that match this signer name. - Mutually-exclusive with name. The contents of all selected - ClusterTrustBundles will be unified and deduplicated. - type: string - required: - - path - type: object - configMap: - description: configMap information about - the configMap data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key - to a path within a volume. - properties: - key: - description: key is the key - to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional specify whether - the ConfigMap or its keys must - be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - downwardAPI: - description: downwardAPI information - about the downwardAPI data to project - properties: - items: - description: Items is a list of - DownwardAPIVolume file - items: - description: DownwardAPIVolumeFile - represents information to create - the file containing the pod - field - properties: - fieldRef: - description: 'Required: Selects - a field of the pod: only - annotations, labels, name, - namespace and uid are supported.' - properties: - apiVersion: - description: Version of - the schema the FieldPath - is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the - field to select in the - specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path - is the relative path name - of the file to be created. - Must not be absolute or - contain the ''..'' path. - Must be utf-8 encoded. The - first item of the relative - path must not start with - ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container - name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies - the output format of - the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: - resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - secret: - description: secret information about - the secret data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key - to a path within a volume. - properties: - key: - description: key is the key - to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional field specify - whether the Secret or its key - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - serviceAccountToken: - description: serviceAccountToken is - information about the serviceAccountToken - data to project - properties: - audience: - description: |- - audience is the intended audience of the token. A recipient of a token - must identify itself with an identifier specified in the audience of the - token, and otherwise should reject the token. The audience defaults to the - identifier of the apiserver. - type: string - expirationSeconds: - description: |- - expirationSeconds is the requested duration of validity of the service - account token. As the token approaches expiration, the kubelet volume - plugin will proactively rotate the service account token. The kubelet will - start trying to rotate the token if the token is older than 80 percent of - its time to live or if the token is older than 24 hours.Defaults to 1 hour - and must be at least 10 minutes. - format: int64 - type: integer - path: - description: |- - path is the path relative to the mount point of the file to project the - token into. - type: string - required: - - path - type: object - type: object - type: array - x-kubernetes-list-type: atomic - type: object - quobyte: - description: quobyte represents a Quobyte mount - on the host that shares a pod's lifetime - properties: - group: - description: |- - group to map volume access to - Default is no group - type: string - readOnly: - description: |- - readOnly here will force the Quobyte volume to be mounted with read-only permissions. - Defaults to false. - type: boolean - registry: - description: |- - registry represents a single or multiple Quobyte Registry services - specified as a string as host:port pair (multiple entries are separated with commas) - which acts as the central registry for volumes - type: string - tenant: - description: |- - tenant owning the given Quobyte volume in the Backend - Used with dynamically provisioned Quobyte volumes, value is set by the plugin - type: string - user: - description: |- - user to map volume access to - Defaults to serivceaccount user - type: string - volume: - description: volume is a string that references - an already created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: |- - rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. - More info: https://examples.k8s.io/volumes/rbd/README.md - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - type: string - image: - description: |- - image is the rados image name. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - keyring: - default: /etc/ceph/keyring - description: |- - keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - monitors: - description: |- - monitors is a collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - items: - type: string - type: array - x-kubernetes-list-type: atomic - pool: - default: rbd - description: |- - pool is the rados pool name. - Default is rbd. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: boolean - secretRef: - description: |- - secretRef is name of the authentication secret for RBDUser. If provided - overrides keyring. - Default is nil. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - user: - default: admin - description: |- - user is the rados user name. - Default is admin. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - required: - - image - - monitors - type: object - scaleIO: - description: scaleIO represents a ScaleIO persistent - volume attached and mounted on Kubernetes nodes. - properties: - fsType: - default: xfs - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". - Default is "xfs". - type: string - gateway: - description: gateway is the host address of - the ScaleIO API Gateway. - type: string - protectionDomain: - description: protectionDomain is the name - of the ScaleIO Protection Domain for the - configured storage. - type: string - readOnly: - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef references to the secret for ScaleIO user and other - sensitive information. If this is not provided, Login operation will fail. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - sslEnabled: - description: sslEnabled Flag enable/disable - SSL communication with Gateway, default - false - type: boolean - storageMode: - default: ThinProvisioned - description: |- - storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage - Pool associated with the protection domain. - type: string - system: - description: system is the name of the storage - system as configured in ScaleIO. - type: string - volumeName: - description: |- - volumeName is the name of a volume already created in the ScaleIO system - that is associated with this volume source. - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - description: |- - secret represents a secret that should populate this volume. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - properties: - defaultMode: - description: |- - defaultMode is Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values - for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items If unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path - within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - optional: - description: optional field specify whether - the Secret or its keys must be defined - type: boolean - secretName: - description: |- - secretName is the name of the secret in the pod's namespace to use. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - type: string - type: object - storageos: - description: storageOS represents a StorageOS - volume attached and mounted on Kubernetes nodes. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef specifies the secret to use for obtaining the StorageOS API - credentials. If not specified, default values will be attempted. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - volumeName: - description: |- - volumeName is the human-readable name of the StorageOS volume. Volume - names are only unique within a namespace. - type: string - volumeNamespace: - description: |- - volumeNamespace specifies the scope of the volume within StorageOS. If no - namespace is specified then the Pod's namespace will be used. This allows the - Kubernetes name scoping to be mirrored within StorageOS for tighter integration. - Set VolumeName to any name to override the default behaviour. - Set to "default" if you are not using namespaces within StorageOS. - Namespaces that do not pre-exist within StorageOS will be created. - type: string - type: object - vsphereVolume: - description: vsphereVolume represents a vSphere - volume attached and mounted on kubelets host - machine - properties: - fsType: - description: |- - fsType is filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage - Policy Based Management (SPBM) profile ID - associated with the StoragePolicyName. - type: string - storagePolicyName: - description: storagePolicyName is the storage - Policy Based Management (SPBM) profile name. - type: string - volumePath: - description: volumePath is the path that identifies - vSphere volume vmdk - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - containers - type: object - type: object - topologyRequest: - description: topologyRequest defines the topology request for - the PodSet. - properties: - podIndexLabel: - description: |- - PodIndexLabel indicates the name of the label indexing the pods. - For example, in the context of - - kubernetes job this is: kubernetes.io/job-completion-index - - JobSet: kubernetes.io/job-completion-index (inherited from Job) - - Kubeflow: training.kubeflow.org/replica-index - type: string - preferred: - description: |- - preferred indicates the topology level preferred by the PodSet, as - indicated by the `kueue.x-k8s.io/podset-preferred-topology` PodSet - annotation. - type: string - required: - description: |- - required indicates the topology level required by the PodSet, as - indicated by the `kueue.x-k8s.io/podset-required-topology` PodSet - annotation. - type: string - subGroupCount: - description: |- - SubGroupIndexLabel indicates the count of replicated Jobs (groups) within a PodSet. - For example, in the context of JobSet this value is read from jobset.sigs.k8s.io/replicatedjob-replicas. - format: int32 - type: integer - subGroupIndexLabel: - description: |- - SubGroupIndexLabel indicates the name of the label indexing the instances of replicated Jobs (groups) - within a PodSet. For example, in the context of JobSet this is jobset.sigs.k8s.io/job-index. - type: string - type: object - required: - - count - - template - type: object - x-kubernetes-validations: - - message: minCount should be positive and less or equal to count - rule: 'has(self.minCount) ? self.minCount <= self.count : true' - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - priority: - description: |- - Priority determines the order of access to the resources managed by the - ClusterQueue where the workload is queued. - The priority value is populated from PriorityClassName. - The higher the value, the higher the priority. - If priorityClassName is specified, priority must not be null. - format: int32 - type: integer - priorityClassName: - description: |- - If specified, indicates the workload's priority. - "system-node-critical" and "system-cluster-critical" are two special - keywords which indicate the highest priorities with the former being - the highest priority. Any other name must be defined by creating a - PriorityClass object with that name. If not specified, the workload - priority will be default or zero if there is no default. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - priorityClassSource: - default: "" - description: |- - priorityClassSource determines whether the priorityClass field refers to a pod PriorityClass or kueue.x-k8s.io/workloadpriorityclass. - Workload's PriorityClass can accept the name of a pod priorityClass or a workloadPriorityClass. - When using pod PriorityClass, a priorityClassSource field has the scheduling.k8s.io/priorityclass value. - enum: - - kueue.x-k8s.io/workloadpriorityclass - - scheduling.k8s.io/priorityclass - - "" - type: string - queueName: - description: |- - queueName is the name of the LocalQueue the Workload is associated with. - queueName cannot be changed while .status.admission is not null. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - podSets - type: object - x-kubernetes-validations: - - message: priority should not be nil when priorityClassName is set - rule: 'has(self.priorityClassName) ? has(self.priority) : true' - status: - description: WorkloadStatus defines the observed state of Workload - properties: - accumulatedPastExexcutionTimeSeconds: - description: |- - accumulatedPastExexcutionTimeSeconds holds the total time, in seconds, the workload spent - in Admitted state, in the previous `Admit` - `Evict` cycles. - format: int32 - type: integer - admission: - description: |- - admission holds the parameters of the admission of the workload by a - ClusterQueue. admission can be set back to null, but its fields cannot be - changed once set. - properties: - clusterQueue: - description: clusterQueue is the name of the ClusterQueue that - admitted this workload. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - podSetAssignments: - description: PodSetAssignments hold the admission results for - each of the .spec.podSets entries. - items: - properties: - count: - description: |- - count is the number of pods taken into account at admission time. - This field will not change in case of quota reclaim. - Value could be missing for Workloads created before this field was added, - in that case spec.podSets[*].count value will be used. - format: int32 - minimum: 0 - type: integer - flavors: - additionalProperties: - description: ResourceFlavorReference is the name of the - ResourceFlavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - description: Flavors are the flavors assigned to the workload - for each resource. - type: object - name: - default: main - description: Name is the name of the podSet. It should match - one of the names in .spec.podSets. - maxLength: 63 - pattern: ^(?i)[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - resourceUsage: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - resourceUsage keeps track of the total resources all the pods in the podset need to run. - - Beside what is provided in podSet's specs, this calculation takes into account - the LimitRange defaults and RuntimeClass overheads at the moment of admission. - This field will not change in case of quota reclaim. - type: object - topologyAssignment: - description: |- - topologyAssignment indicates the topology assignment divided into - topology domains corresponding to the lowest level of the topology. - The assignment specifies the number of Pods to be scheduled per topology - domain and specifies the node selectors for each topology domain, in the - following way: the node selector keys are specified by the levels field - (same for all domains), and the corresponding node selector value is - specified by the domains.values subfield. If the TopologySpec.Levels field contains - "kubernetes.io/hostname" label, topologyAssignment will contain data only for - this label, and omit higher levels in the topology - - Example: - - topologyAssignment: - levels: - - cloud.provider.com/topology-block - - cloud.provider.com/topology-rack - domains: - - values: [block-1, rack-1] - count: 4 - - values: [block-1, rack-2] - count: 2 - - Here: - - 4 Pods are to be scheduled on nodes matching the node selector: - cloud.provider.com/topology-block: block-1 - cloud.provider.com/topology-rack: rack-1 - - 2 Pods are to be scheduled on nodes matching the node selector: - cloud.provider.com/topology-block: block-1 - cloud.provider.com/topology-rack: rack-2 - - Example: - Below there is an equivalent of the above example assuming, Topology - object defines kubernetes.io/hostname as the lowest level in topology. - Hence we omit higher level of topologies, since the hostname label - is sufficient to explicitly identify a proper node. - - topologyAssignment: - levels: - - kubernetes.io/hostname - domains: - - values: [hostname-1] - count: 4 - - values: [hostname-2] - count: 2 - properties: - domains: - description: |- - domains is a list of topology assignments split by topology domains at - the lowest level of the topology. - items: - properties: - count: - description: |- - count indicates the number of Pods to be scheduled in the topology - domain indicated by the values field. - format: int32 - minimum: 1 - type: integer - values: - description: |- - values is an ordered list of node selector values describing a topology - domain. The values correspond to the consecutive topology levels, from - the highest to the lowest. - items: - type: string - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - required: - - count - - values - type: object - type: array - levels: - description: |- - levels is an ordered list of keys denoting the levels of the assigned - topology (i.e. node label keys), from the highest to the lowest level of - the topology. - items: - type: string - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - required: - - domains - - levels - type: object - required: - - name - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - clusterQueue - - podSetAssignments - type: object - admissionChecks: - description: admissionChecks list all the admission checks required - by the workload and the current status - items: - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - name: - description: name identifies the admission check. - maxLength: 316 - type: string - podSetUpdates: - items: - description: |- - PodSetUpdate contains a list of pod set modifications suggested by AdmissionChecks. - The modifications should be additive only - modifications of already existing keys - or having the same key provided by multiple AdmissionChecks is not allowed and will - result in failure during workload admission. - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - name: - description: Name of the PodSet to modify. Should match - to one of the Workload's PodSets. - type: string - nodeSelector: - additionalProperties: - type: string - type: object - tolerations: - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - maxItems: 8 - type: array - x-kubernetes-validations: - - message: operator must be Exists when 'key' is empty, - which means 'match all values and all keys' - rule: 'self.all(x, !has(x.key) ? x.operator == ''Exists'' - : true)' - - message: effect must be 'NoExecute' when 'tolerationSeconds' - is set - rule: 'self.all(x, has(x.tolerationSeconds) ? x.effect - == ''NoExecute'' : true)' - - message: 'supported toleration values: ''Equal''(default), - ''Exists''' - rule: self.all(x, !has(x.operator) || x.operator in - ['Equal', 'Exists']) - - message: a value must be empty when 'operator' is 'Exists' - rule: 'self.all(x, has(x.operator) && x.operator == - ''Exists'' ? !has(x.value) : true)' - - message: 'supported taint effect values: ''NoSchedule'', - ''PreferNoSchedule'', ''NoExecute''' - rule: self.all(x, !has(x.effect) || x.effect in ['NoSchedule', - 'PreferNoSchedule', 'NoExecute']) - required: - - name - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - state: - description: state of the admissionCheck, one of Pending, Ready, - Retry, Rejected - enum: - - Pending - - Ready - - Retry - - Rejected - type: string - required: - - lastTransitionTime - - message - - name - - state - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - conditions: - description: |- - conditions hold the latest available observations of the Workload - current state. - - The type of the condition could be: - - - Admitted: the Workload was admitted through a ClusterQueue. - - Finished: the associated workload finished running (failed or succeeded). - - PodsReady: at least `.spec.podSets[*].count` Pods are ready or have - succeeded. - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - reclaimablePods: - description: |- - reclaimablePods keeps track of the number pods within a podset for which - the resource reservation is no longer needed. - items: - properties: - count: - description: count is the number of pods for which the requested - resources are no longer needed. - format: int32 - minimum: 0 - type: integer - name: - description: name is the PodSet name. - type: string - required: - - count - - name - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - requeueState: - description: |- - requeueState holds the re-queue state - when a workload meets Eviction with PodsReadyTimeout reason. - properties: - count: - description: |- - count records the number of times a workload has been re-queued - When a deactivated (`.spec.activate`=`false`) workload is reactivated (`.spec.activate`=`true`), - this count would be reset to null. - format: int32 - minimum: 0 - type: integer - requeueAt: - description: |- - requeueAt records the time when a workload will be re-queued. - When a deactivated (`.spec.activate`=`false`) workload is reactivated (`.spec.activate`=`true`), - this time would be reset to null. - format: date-time - type: string - type: object - resourceRequests: - description: |- - resourceRequests provides a detailed view of the resources that were - requested by a non-admitted workload when it was considered for admission. - If admission is non-null, resourceRequests will be empty because - admission.resourceUsage contains the detailed information. - items: - properties: - name: - default: main - description: name is the name of the podSet. It should match - one of the names in .spec.podSets. - maxLength: 63 - pattern: ^(?i)[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - resources: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - resources is the total resources all the pods in the podset need to run. - - Beside what is provided in podSet's specs, this value also takes into account - the LimitRange defaults and RuntimeClass overheads at the moment of consideration - and the application of resource.excludeResourcePrefixes and resource.transformations. - type: object - required: - - name - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - type: object - x-kubernetes-validations: - - message: podSetAssignments must have the same number of podSets as the spec - rule: 'has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'') && has(self.status.admission) - ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) - : true' - - message: field is immutable - rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'')) ? (oldSelf.spec.priorityClassSource - == self.spec.priorityClassSource) : true' - - message: field is immutable - rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassName) - && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName - == self.spec.priorityClassName) : true' - - message: field is immutable - rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'')) && (has(self.status) - && has(self.status.conditions) && self.status.conditions.exists(c, c.type - == ''QuotaReserved'' && c.status == ''True'')) && has(oldSelf.spec.queueName) - && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName - : true' - - message: maximumExecutionTimeSeconds is immutable while admitted - rule: ((has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == 'Admitted' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) - && self.status.conditions.exists(c, c.type == 'Admitted' && c.status == - 'True')))?((has(oldSelf.spec.maximumExecutionTimeSeconds)?oldSelf.spec.maximumExecutionTimeSeconds:0) - == (has(self.spec.maximumExecutionTimeSeconds)?self.spec.maximumExecutionTimeSeconds:0)):true - served: true - storage: true - subresources: - status: {} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-controller-manager - namespace: kueue-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-leader-election-role - namespace: kueue-system -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch ---- -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.kueue.x-k8s.io/batch-admin: "true" -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-batch-admin-role ---- -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.kueue.x-k8s.io/batch-user: "true" -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-batch-user-role ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-clusterqueue-editor-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - clusterqueues - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - clusterqueues/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-clusterqueue-viewer-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - clusterqueues - verbs: - - get - - list - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - clusterqueues/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-job-editor-role -rules: -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-job-viewer-role -rules: -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch -- apiGroups: - - batch - resources: - - jobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-jobset-editor-role -rules: -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-jobset-viewer-role -rules: -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets - verbs: - - get - - list - - watch -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-localqueue-editor-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - localqueues - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - localqueues/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-localqueue-viewer-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - localqueues - verbs: - - get - - list - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - localqueues/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-manager-role -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - watch -- apiGroups: - - "" - resources: - - limitranges - - namespaces - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods/finalizers - verbs: - - get - - update -- apiGroups: - - "" - resources: - - pods/status - verbs: - - get - - patch -- apiGroups: - - "" - resources: - - podtemplates - verbs: - - create - - delete - - get - - list - - update - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - update - - watch -- apiGroups: - - admissionregistration.k8s.io - resources: - - mutatingwebhookconfigurations - - validatingwebhookconfigurations - verbs: - - get - - list - - update - - watch -- apiGroups: - - apps - resources: - - replicasets - - statefulsets - verbs: - - get - - list - - watch -- apiGroups: - - autoscaling.x-k8s.io - resources: - - provisioningrequests - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - autoscaling.x-k8s.io - resources: - - provisioningrequests/status - verbs: - - get -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs/finalizers - - jobs/status - verbs: - - get - - patch - - update -- apiGroups: - - flowcontrol.apiserver.k8s.io - resources: - - flowschemas - - prioritylevelconfigurations - verbs: - - list - - watch -- apiGroups: - - flowcontrol.apiserver.k8s.io - resources: - - flowschemas/status - verbs: - - patch -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/finalizers - verbs: - - get - - update -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/status - verbs: - - get - - patch - - update -- apiGroups: - - kubeflow.org - resources: - - mpijobs - - mxjobs - - paddlejobs - - pytorchjobs - - tfjobs - - xgboostjobs - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - mpijobs/finalizers - - mxjobs/finalizers - - mxjobs/status - - paddlejobs/finalizers - - pytorchjobs/finalizers - - tfjobs/finalizers - - xgboostjobs/finalizers - verbs: - - get - - update -- apiGroups: - - kubeflow.org - resources: - - mpijobs/status - - paddlejobs/status - - pytorchjobs/status - - tfjobs/status - - xgboostjobs/status - verbs: - - get - - patch - - update -- apiGroups: - - kueue.x-k8s.io - resources: - - admissionchecks - - clusterqueues - - cohorts - - localqueues - - workloads - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - admissionchecks/finalizers - - clusterqueues/finalizers - - localqueues/finalizers - - resourceflavors/finalizers - - topologies/finalizers - - workloads/finalizers - verbs: - - update -- apiGroups: - - kueue.x-k8s.io - resources: - - admissionchecks/status - - clusterqueues/status - - localqueues/status - - multikueueclusters/status - - workloads/status - verbs: - - get - - patch - - update -- apiGroups: - - kueue.x-k8s.io - resources: - - multikueueclusters - - multikueueconfigs - - provisioningrequestconfigs - - workloadpriorityclasses - verbs: - - get - - list - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - resourceflavors - verbs: - - delete - - get - - list - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - topologies - verbs: - - get - - list - - update - - watch -- apiGroups: - - node.k8s.io - resources: - - runtimeclasses - verbs: - - get - - list - - watch -- apiGroups: - - ray.io - resources: - - rayclusters - - rayjobs - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - ray.io - resources: - - rayclusters/finalizers - - rayclusters/status - - rayjobs/finalizers - - rayjobs/status - verbs: - - get - - update -- apiGroups: - - scheduling.k8s.io - resources: - - priorityclasses - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-metrics-reader -rules: -- nonResourceURLs: - - /metrics - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-mpijob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - mpijobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - mpijobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-mpijob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - mpijobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - mpijobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-mxjob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - mxjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - mxjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-mxjob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - mxjobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - mxjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-paddlejob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - paddlejobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - paddlejobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-paddlejob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - paddlejobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - paddlejobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-pending-workloads-cq-viewer-role -rules: -- apiGroups: - - visibility.kueue.x-k8s.io - resources: - - clusterqueues/pendingworkloads - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-pending-workloads-lq-viewer-role -rules: -- apiGroups: - - visibility.kueue.x-k8s.io - resources: - - localqueues/pendingworkloads - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-pytorchjob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-pytorchjob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-raycluster-editor-role -rules: -- apiGroups: - - ray.io - resources: - - rayclusters - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - ray.io - resources: - - rayclusters/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-raycluster-viewer-role -rules: -- apiGroups: - - ray.io - resources: - - rayclusters - verbs: - - get - - list - - watch -- apiGroups: - - ray.io - resources: - - rayclusters/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-rayjob-editor-role -rules: -- apiGroups: - - ray.io - resources: - - rayjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - ray.io - resources: - - rayjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-rayjob-viewer-role -rules: -- apiGroups: - - ray.io - resources: - - rayjobs - verbs: - - get - - list - - watch -- apiGroups: - - ray.io - resources: - - rayjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-resourceflavor-editor-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - resourceflavors - verbs: - - create - - delete - - get - - list - - patch - - update - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-resourceflavor-viewer-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - resourceflavors - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-tfjob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - tfjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - tfjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-tfjob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - tfjobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - tfjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-workload-editor-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - workloads - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - workloads/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-workload-viewer-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - workloads - verbs: - - get - - list - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - workloads/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-xgboostjob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - xgboostjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - xgboostjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-xgboostjob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - xgboostjobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - xgboostjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-visibility-server-auth-reader - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader -subjects: -- kind: ServiceAccount - name: kueue-controller-manager - namespace: kueue-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-leader-election-rolebinding - namespace: kueue-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: kueue-leader-election-role -subjects: -- kind: ServiceAccount - name: kueue-controller-manager - namespace: kueue-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kueue-manager-role -subjects: -- kind: ServiceAccount - name: kueue-controller-manager - namespace: kueue-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kueue-proxy-role -subjects: -- kind: ServiceAccount - name: kueue-controller-manager - namespace: kueue-system ---- -apiVersion: v1 -data: - controller_manager_config.yaml: | - apiVersion: config.kueue.x-k8s.io/v1beta1 - kind: Configuration - health: - healthProbeBindAddress: :8081 - metrics: - bindAddress: :8080 - # enableClusterQueueResources: true - webhook: - port: 9443 - leaderElection: - leaderElect: true - resourceName: c1f6bfd2.kueue.x-k8s.io - controller: - groupKindConcurrency: - Job.batch: 5 - Pod: 5 - Workload.kueue.x-k8s.io: 5 - LocalQueue.kueue.x-k8s.io: 1 - Cohort.kueue.x-k8s.io: 1 - ClusterQueue.kueue.x-k8s.io: 1 - ResourceFlavor.kueue.x-k8s.io: 1 - clientConnection: - qps: 50 - burst: 100 - #pprofBindAddress: :8083 - #waitForPodsReady: - # enable: false - # timeout: 5m - # blockAdmission: false - # requeuingStrategy: - # timestamp: Eviction - # backoffLimitCount: null # null indicates infinite requeuing - # backoffBaseSeconds: 60 - # backoffMaxSeconds: 3600 - #manageJobsWithoutQueueName: true - #managedJobsNamespaceSelector: - # matchLabels: - # kueue-managed: "true" - #internalCertManagement: - # enable: false - # webhookServiceName: "" - # webhookSecretName: "" - integrations: - frameworks: - - "batch/job" - - "kubeflow.org/mpijob" - - "ray.io/rayjob" - - "ray.io/raycluster" - - "jobset.x-k8s.io/jobset" - - "kubeflow.org/mxjob" - - "kubeflow.org/paddlejob" - - "kubeflow.org/pytorchjob" - - "kubeflow.org/tfjob" - - "kubeflow.org/xgboostjob" - # - "pod" - # - "deployment" # requires enabling pod integration - # - "statefulset" # requires enabling pod integration - # externalFrameworks: - # - "Foo.v1.example.com" - # podOptions: - # namespaceSelector: - # matchExpressions: - # - key: kubernetes.io/metadata.name - # operator: NotIn - # values: [ kube-system, kueue-system ] - #fairSharing: - # enable: true - # preemptionStrategies: [LessThanOrEqualToFinalShare, LessThanInitialShare] - #resources: - # excludeResourcePrefixes: [] - # transformations: - # - input: nvidia.com/mig-4g.5gb - # strategy: Replace | Retain - # outputs: - # example.com/accelerator-memory: 5Gi - # example.com/accelerator-gpc: 4 -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-manager-config - namespace: kueue-system ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-webhook-server-cert - namespace: kueue-system ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-controller-manager-metrics-service - namespace: kueue-system -spec: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https - selector: - control-plane: controller-manager ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-visibility-server - namespace: kueue-system -spec: - ports: - - name: https - port: 443 - protocol: TCP - targetPort: 8082 - selector: - control-plane: controller-manager ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-webhook-service - namespace: kueue-system -spec: - ports: - - port: 443 - protocol: TCP - targetPort: 9443 - selector: - control-plane: controller-manager ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-controller-manager - namespace: kueue-system -spec: - replicas: 1 - selector: - matchLabels: - control-plane: controller-manager - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: manager - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - spec: - containers: - - args: - - --config=/controller_manager_config.yaml - - --zap-log-level=2 - - --feature-gates=TopologyAwareScheduling=true - command: - - /manager - image: registry.k8s.io/kueue/kueue:v0.10.1 - imagePullPolicy: Always - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - ports: - - containerPort: 8082 - name: visibility - protocol: TCP - - containerPort: 9443 - name: webhook-server - protocol: TCP - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 500m - memory: 512Mi - securityContext: - allowPrivilegeEscalation: false - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true - - mountPath: /controller_manager_config.yaml - name: manager-config - subPath: controller_manager_config.yaml - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=10 - image: registry.k8s.io/kubebuilder/kube-rbac-proxy:v0.16.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - securityContext: - runAsNonRoot: true - serviceAccountName: kueue-controller-manager - terminationGracePeriodSeconds: 10 - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: kueue-webhook-server-cert - - configMap: - name: kueue-manager-config - name: manager-config - tolerations: - - effect: NoSchedule - key: components.gke.io/gke-managed-components - operator: Equal - value: "true" ---- -apiVersion: apiregistration.k8s.io/v1 -kind: APIService -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: v1beta1.visibility.kueue.x-k8s.io -spec: - group: visibility.kueue.x-k8s.io - groupPriorityMinimum: 100 - insecureSkipTLSVerify: true - service: - name: kueue-visibility-server - namespace: kueue-system - version: v1beta1 - versionPriority: 100 ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-mutating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate--v1-pod - failurePolicy: Fail - name: mpod.kb.io - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kueue-system - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-apps-v1-deployment - failurePolicy: Fail - name: mdeployment.kb.io - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kueue-system - rules: - - apiGroups: - - apps - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - deployments - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-batch-v1-job - failurePolicy: Fail - name: mjob.kb.io - rules: - - apiGroups: - - batch - apiVersions: - - v1 - operations: - - CREATE - resources: - - jobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-jobset-x-k8s-io-v1alpha2-jobset - failurePolicy: Fail - name: mjobset.kb.io - rules: - - apiGroups: - - jobset.x-k8s.io - apiVersions: - - v1alpha2 - operations: - - CREATE - resources: - - jobsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v1-mxjob - failurePolicy: Fail - name: mmxjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - resources: - - mxjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v1-paddlejob - failurePolicy: Fail - name: mpaddlejob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - resources: - - paddlejobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v1-pytorchjob - failurePolicy: Fail - name: mpytorchjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - resources: - - pytorchjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v1-tfjob - failurePolicy: Fail - name: mtfjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - resources: - - tfjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v1-xgboostjob - failurePolicy: Fail - name: mxgboostjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - resources: - - xgboostjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v2beta1-mpijob - failurePolicy: Fail - name: mmpijob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v2beta1 - operations: - - CREATE - resources: - - mpijobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-ray-io-v1-raycluster - failurePolicy: Fail - name: mraycluster.kb.io - rules: - - apiGroups: - - ray.io - apiVersions: - - v1 - operations: - - CREATE - resources: - - rayclusters - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-ray-io-v1-rayjob - failurePolicy: Fail - name: mrayjob.kb.io - rules: - - apiGroups: - - ray.io - apiVersions: - - v1 - operations: - - CREATE - resources: - - rayjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-apps-v1-statefulset - failurePolicy: Fail - name: mstatefulset.kb.io - rules: - - apiGroups: - - apps - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - statefulsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kueue-x-k8s-io-v1beta1-clusterqueue - failurePolicy: Fail - name: mclusterqueue.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - resources: - - clusterqueues - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kueue-x-k8s-io-v1beta1-resourceflavor - failurePolicy: Fail - name: mresourceflavor.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - resources: - - resourceflavors - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kueue-x-k8s-io-v1beta1-workload - failurePolicy: Fail - name: mworkload.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - resources: - - workloads - sideEffects: None ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-validating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate--v1-pod - failurePolicy: Fail - name: vpod.kb.io - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kueue-system - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - pods - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-apps-v1-deployment - failurePolicy: Fail - name: vdeployment.kb.io - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kueue-system - rules: - - apiGroups: - - apps - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - deployments - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-batch-v1-job - failurePolicy: Fail - name: vjob.kb.io - rules: - - apiGroups: - - batch - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - jobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-jobset-x-k8s-io-v1alpha2-jobset - failurePolicy: Fail - name: vjobset.kb.io - rules: - - apiGroups: - - jobset.x-k8s.io - apiVersions: - - v1alpha2 - operations: - - CREATE - - UPDATE - resources: - - jobsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v1-mxjob - failurePolicy: Fail - name: vmxjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - mxjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v1-paddlejob - failurePolicy: Fail - name: vpaddlejob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - paddlejobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v1-pytorchjob - failurePolicy: Fail - name: vpytorchjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - pytorchjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v1-tfjob - failurePolicy: Fail - name: vtfjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - tfjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v1-xgboostjob - failurePolicy: Fail - name: vxgboostjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - xgboostjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v2beta1-mpijob - failurePolicy: Fail - name: vmpijob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v2beta1 - operations: - - CREATE - - UPDATE - resources: - - mpijobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-ray-io-v1-raycluster - failurePolicy: Fail - name: vraycluster.kb.io - rules: - - apiGroups: - - ray.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - rayclusters - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-ray-io-v1-rayjob - failurePolicy: Fail - name: vrayjob.kb.io - rules: - - apiGroups: - - ray.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - rayjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-apps-v1-statefulset - failurePolicy: Fail - name: vstatefulset.kb.io - rules: - - apiGroups: - - apps - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - statefulsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kueue-x-k8s-io-v1beta1-clusterqueue - failurePolicy: Fail - name: vclusterqueue.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - clusterqueues - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kueue-x-k8s-io-v1alpha1-cohort - failurePolicy: Fail - name: vcohort.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1alpha1 - operations: - - CREATE - - UPDATE - resources: - - cohorts - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kueue-x-k8s-io-v1beta1-resourceflavor - failurePolicy: Fail - name: vresourceflavor.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - resourceflavors - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kueue-x-k8s-io-v1beta1-workload - failurePolicy: Fail - name: vworkload.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - workloads - - workloads/status - sideEffects: None diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml deleted file mode 100644 index ca1e208d34..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.11.4.yaml +++ /dev/null @@ -1,13466 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Namespace -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-system ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.17.2 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: admissionchecks.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: AdmissionCheck - listKind: AdmissionCheckList - plural: admissionchecks - singular: admissioncheck - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: AdmissionCheck is the Schema for the admissionchecks API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: AdmissionCheckSpec defines the desired state of AdmissionCheck - properties: - controllerName: - description: |- - controllerName identifies the controller that processes the AdmissionCheck, - not necessarily a Kubernetes Pod or Deployment name. Cannot be empty. - type: string - x-kubernetes-validations: - - message: field is immutable - rule: self == oldSelf - parameters: - description: |- - Parameters identifies a configuration with additional parameters for the - check. - properties: - apiGroup: - description: ApiGroup is the group for the resource being referenced. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: Kind is the type of the resource being referenced. - maxLength: 63 - pattern: ^(?i)[a-z]([-a-z0-9]*[a-z0-9])?$ - type: string - name: - description: Name is the name of the resource being referenced. - maxLength: 63 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - apiGroup - - kind - - name - type: object - retryDelayMinutes: - default: 15 - description: |- - RetryDelayMinutes specifies how long to keep the workload suspended after - a failed check (after it transitioned to False). When the delay period has passed, the check - state goes to "Unknown". The default is 15 min. - Deprecated: retryDelayMinutes has already been deprecated since v0.8 and will be removed in v1beta2. - format: int64 - type: integer - required: - - controllerName - type: object - status: - description: AdmissionCheckStatus defines the observed state of AdmissionCheck - properties: - conditions: - description: |- - conditions hold the latest available observations of the AdmissionCheck - current state. - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - controller-gen.kubebuilder.io/version: v0.17.2 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: clusterqueues.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: ClusterQueue - listKind: ClusterQueueList - plural: clusterqueues - shortNames: - - cq - singular: clusterqueue - scope: Cluster - versions: - - additionalPrinterColumns: - - description: Cohort that this ClusterQueue belongs to - jsonPath: .spec.cohort - name: Cohort - type: string - - description: The queueing strategy used to prioritize workloads - jsonPath: .spec.queueingStrategy - name: Strategy - priority: 1 - type: string - - description: Number of pending workloads - jsonPath: .status.pendingWorkloads - name: Pending Workloads - type: integer - - description: Number of admitted workloads that haven't finished yet - jsonPath: .status.admittedWorkloads - name: Admitted Workloads - priority: 1 - type: integer - name: v1beta1 - schema: - openAPIV3Schema: - description: ClusterQueue is the Schema for the clusterQueue API. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: ClusterQueueSpec defines the desired state of ClusterQueue - properties: - admissionChecks: - description: |- - admissionChecks lists the AdmissionChecks required by this ClusterQueue. - Cannot be used along with AdmissionCheckStrategy. - items: - type: string - type: array - admissionChecksStrategy: - description: |- - admissionCheckStrategy defines a list of strategies to determine which ResourceFlavors require AdmissionChecks. - This property cannot be used in conjunction with the 'admissionChecks' property. - properties: - admissionChecks: - description: admissionChecks is a list of strategies for AdmissionChecks - items: - description: AdmissionCheckStrategyRule defines rules for a - single AdmissionCheck - properties: - name: - description: name is an AdmissionCheck's name. - type: string - onFlavors: - description: |- - onFlavors is a list of ResourceFlavors' names that this AdmissionCheck should run for. - If empty, the AdmissionCheck will run for all workloads submitted to the ClusterQueue. - items: - description: ResourceFlavorReference is the name of the - ResourceFlavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - type: array - required: - - name - type: object - type: array - type: object - cohort: - description: |- - cohort that this ClusterQueue belongs to. CQs that belong to the - same cohort can borrow unused resources from each other. - - A CQ can be a member of a single borrowing cohort. A workload submitted - to a queue referencing this CQ can borrow quota from any CQ in the cohort. - Only quota for the [resource, flavor] pairs listed in the CQ can be - borrowed. - If empty, this ClusterQueue cannot borrow from any other ClusterQueue and - vice versa. - - A cohort is a name that links CQs together, but it doesn't reference any - object. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - fairSharing: - description: |- - fairSharing defines the properties of the ClusterQueue when - participating in FairSharing. The values are only relevant - if FairSharing is enabled in the Kueue configuration. - properties: - weight: - anyOf: - - type: integer - - type: string - default: 1 - description: |- - weight gives a comparative advantage to this ClusterQueue - or Cohort when competing for unused resources in the - Cohort. The share is based on the dominant resource usage - above nominal quotas for each resource, divided by the - weight. Admission prioritizes scheduling workloads from - ClusterQueues and Cohorts with the lowest share and - preempting workloads from the ClusterQueues and Cohorts - with the highest share. A zero weight implies infinite - share value, meaning that this Node will always be at - disadvantage against other ClusterQueues and Cohorts. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - flavorFungibility: - default: {} - description: |- - flavorFungibility defines whether a workload should try the next flavor - before borrowing or preempting in the flavor being evaluated. - properties: - whenCanBorrow: - default: Borrow - description: |- - whenCanBorrow determines whether a workload should try the next flavor - before borrowing in current flavor. The possible values are: - - - `Borrow` (default): allocate in current flavor if borrowing - is possible. - - `TryNextFlavor`: try next flavor even if the current - flavor has enough resources to borrow. - enum: - - Borrow - - TryNextFlavor - type: string - whenCanPreempt: - default: TryNextFlavor - description: |- - whenCanPreempt determines whether a workload should try the next flavor - before borrowing in current flavor. The possible values are: - - - `Preempt`: allocate in current flavor if it's possible to preempt some workloads. - - `TryNextFlavor` (default): try next flavor even if there are enough - candidates for preemption in the current flavor. - enum: - - Preempt - - TryNextFlavor - type: string - type: object - namespaceSelector: - description: |- - namespaceSelector defines which namespaces are allowed to submit workloads to - this clusterQueue. Beyond this basic support for policy, a policy agent like - Gatekeeper should be used to enforce more advanced policies. - Defaults to null which is a nothing selector (no namespaces eligible). - If set to an empty selector `{}`, then all namespaces are eligible. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - preemption: - default: {} - description: |- - ClusterQueuePreemption contains policies to preempt Workloads from this - ClusterQueue or the ClusterQueue's cohort. - - Preemption may be configured to work in the following scenarios: - - - When a Workload fits within the nominal quota of the ClusterQueue, but - the quota is currently borrowed by other ClusterQueues in the cohort. - We preempt workloads in other ClusterQueues to allow this ClusterQueue to - reclaim its nominal quota. Configured using reclaimWithinCohort. - - When a Workload doesn't fit within the nominal quota of the ClusterQueue - and there are admitted Workloads in the ClusterQueue with lower priority. - Configured using withinClusterQueue. - - When a Workload may fit while both borrowing and preempting - low priority workloads in the Cohort. Configured using borrowWithinCohort. - - When FairSharing is enabled, to maintain fair distribution of - unused resources. See FairSharing documentation. - - The preemption algorithm tries to find a minimal set of Workloads to - preempt to accomomdate the pending Workload, preempting Workloads with - lower priority first. - properties: - borrowWithinCohort: - default: {} - description: |- - BorrowWithinCohort contains configuration which allows to preempt workloads - within cohort while borrowing. It only works with Classical Preemption, - __not__ with Fair Sharing. - properties: - maxPriorityThreshold: - description: |- - maxPriorityThreshold allows to restrict the set of workloads which - might be preempted by a borrowing workload, to only workloads with - priority less than or equal to the specified threshold priority. - When the threshold is not specified, then any workload satisfying the - policy can be preempted by the borrowing workload. - format: int32 - type: integer - policy: - default: Never - description: |- - policy determines the policy for preemption to reclaim quota within cohort while borrowing. - Possible values are: - - `Never` (default): do not allow for preemption, in other - ClusterQueues within the cohort, for a borrowing workload. - - `LowerPriority`: allow preemption, in other ClusterQueues - within the cohort, for a borrowing workload, but only if - the preempted workloads are of lower priority. - enum: - - Never - - LowerPriority - type: string - type: object - reclaimWithinCohort: - default: Never - description: |- - reclaimWithinCohort determines whether a pending Workload can preempt - Workloads from other ClusterQueues in the cohort that are using more than - their nominal quota. The possible values are: - - - `Never` (default): do not preempt Workloads in the cohort. - - `LowerPriority`: **Classic Preemption** if the pending Workload - fits within the nominal quota of its ClusterQueue, only preempt - Workloads in the cohort that have lower priority than the pending - Workload. **Fair Sharing** only preempt Workloads in the cohort that - have lower priority than the pending Workload and that satisfy the - Fair Sharing preemptionStategies. - - `Any`: **Classic Preemption** if the pending Workload fits within - the nominal quota of its ClusterQueue, preempt any Workload in the - cohort, irrespective of priority. **Fair Sharing** preempt Workloads - in the cohort that satisfy the Fair Sharing preemptionStrategies. - enum: - - Never - - LowerPriority - - Any - type: string - withinClusterQueue: - default: Never - description: |- - withinClusterQueue determines whether a pending Workload that doesn't fit - within the nominal quota for its ClusterQueue, can preempt active Workloads in - the ClusterQueue. The possible values are: - - - `Never` (default): do not preempt Workloads in the ClusterQueue. - - `LowerPriority`: only preempt Workloads in the ClusterQueue that have - lower priority than the pending Workload. - - `LowerOrNewerEqualPriority`: only preempt Workloads in the ClusterQueue that - either have a lower priority than the pending workload or equal priority - and are newer than the pending workload. - enum: - - Never - - LowerPriority - - LowerOrNewerEqualPriority - type: string - type: object - x-kubernetes-validations: - - message: reclaimWithinCohort=Never and borrowWithinCohort.Policy!=Never - rule: '!(self.reclaimWithinCohort == ''Never'' && has(self.borrowWithinCohort) - && self.borrowWithinCohort.policy != ''Never'')' - queueingStrategy: - default: BestEffortFIFO - description: |- - QueueingStrategy indicates the queueing strategy of the workloads - across the queues in this ClusterQueue. - Current Supported Strategies: - - - StrictFIFO: workloads are ordered strictly by creation time. - Older workloads that can't be admitted will block admitting newer - workloads even if they fit available quota. - - BestEffortFIFO: workloads are ordered by creation time, - however older workloads that can't be admitted will not block - admitting newer workloads that fit existing quota. - enum: - - StrictFIFO - - BestEffortFIFO - type: string - resourceGroups: - description: |- - resourceGroups describes groups of resources. - Each resource group defines the list of resources and a list of flavors - that provide quotas for these resources. - Each resource and each flavor can only form part of one resource group. - resourceGroups can be up to 16. - items: - properties: - coveredResources: - description: |- - coveredResources is the list of resources covered by the flavors in this - group. - Examples: cpu, memory, vendor.com/gpu. - The list cannot be empty and it can contain up to 16 resources. - items: - description: ResourceName is the name identifying various - resources in a ResourceList. - type: string - maxItems: 16 - minItems: 1 - type: array - flavors: - description: |- - flavors is the list of flavors that provide the resources of this group. - Typically, different flavors represent different hardware models - (e.g., gpu models, cpu architectures) or pricing models (on-demand vs spot - cpus). - Each flavor MUST list all the resources listed for this group in the same - order as the .resources field. - The list cannot be empty and it can contain up to 16 flavors. - items: - properties: - name: - description: |- - name of this flavor. The name should match the .metadata.name of a - ResourceFlavor. If a matching ResourceFlavor does not exist, the - ClusterQueue will have an Active condition set to False. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: |- - resources is the list of quotas for this flavor per resource. - There could be up to 16 resources. - items: - properties: - borrowingLimit: - anyOf: - - type: integer - - type: string - description: |- - borrowingLimit is the maximum amount of quota for the [flavor, resource] - combination that this ClusterQueue is allowed to borrow from the unused - quota of other ClusterQueues in the same cohort. - In total, at a given time, Workloads in a ClusterQueue can consume a - quantity of quota equal to nominalQuota+borrowingLimit, assuming the other - ClusterQueues in the cohort have enough unused quota. - If null, it means that there is no borrowing limit. - If not null, it must be non-negative. - borrowingLimit must be null if spec.cohort is empty. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - lendingLimit: - anyOf: - - type: integer - - type: string - description: |- - lendingLimit is the maximum amount of unused quota for the [flavor, resource] - combination that this ClusterQueue can lend to other ClusterQueues in the same cohort. - In total, at a given time, ClusterQueue reserves for its exclusive use - a quantity of quota equals to nominalQuota - lendingLimit. - If null, it means that there is no lending limit, meaning that - all the nominalQuota can be borrowed by other clusterQueues in the cohort. - If not null, it must be non-negative. - lendingLimit must be null if spec.cohort is empty. - This field is in beta stage and is enabled by default. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - name: - description: name of this resource. - type: string - nominalQuota: - anyOf: - - type: integer - - type: string - description: |- - nominalQuota is the quantity of this resource that is available for - Workloads admitted by this ClusterQueue at a point in time. - The nominalQuota must be non-negative. - nominalQuota should represent the resources in the cluster available for - running jobs (after discounting resources consumed by system components - and pods not managed by kueue). In an autoscaled cluster, nominalQuota - should account for resources that can be provided by a component such as - Kubernetes cluster-autoscaler. - - If the ClusterQueue belongs to a cohort, the sum of the quotas for each - (flavor, resource) combination defines the maximum quantity that can be - allocated by a ClusterQueue in the cohort. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - - nominalQuota - type: object - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - coveredResources - - flavors - type: object - x-kubernetes-validations: - - message: flavors must have the same number of resources as the - coveredResources - rule: self.flavors.all(x, size(x.resources) == size(self.coveredResources)) - maxItems: 16 - type: array - x-kubernetes-list-type: atomic - stopPolicy: - default: None - description: |- - stopPolicy - if set to a value different from None, the ClusterQueue is considered Inactive, no new reservation being - made. - - Depending on its value, its associated workloads will: - - - None - Workloads are admitted - - HoldAndDrain - Admitted workloads are evicted and Reserving workloads will cancel the reservation. - - Hold - Admitted workloads will run to completion and Reserving workloads will cancel the reservation. - enum: - - None - - Hold - - HoldAndDrain - type: string - type: object - x-kubernetes-validations: - - message: borrowingLimit must be nil when cohort is empty - rule: '!has(self.cohort) && has(self.resourceGroups) ? self.resourceGroups.all(rg, - rg.flavors.all(f, f.resources.all(r, !has(r.borrowingLimit)))) : true' - status: - description: ClusterQueueStatus defines the observed state of ClusterQueue - properties: - admittedWorkloads: - description: |- - admittedWorkloads is the number of workloads currently admitted to this - clusterQueue and haven't finished yet. - format: int32 - type: integer - conditions: - description: |- - conditions hold the latest available observations of the ClusterQueue - current state. - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - fairSharing: - description: fairSharing contains the information about the current - status of Fair Sharing. - properties: - weightedShare: - description: |- - WeightedShare represent the maximum of the ratios of usage - above nominal quota to the lendable resources in the - Cohort, among all the resources provided by the Node, and - divided by the weight. If zero, it means that the usage of - the Node is below the nominal quota. If the Node has a - weight of zero and is borrowing, this will return - 9223372036854775807, the maximum possible share value. - format: int64 - type: integer - required: - - weightedShare - type: object - flavorsReservation: - description: |- - flavorsReservation are the reserved quotas, by flavor, currently in use by the - workloads assigned to this ClusterQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: resources lists the quota usage for the resources - in this flavor. - items: - properties: - borrowed: - anyOf: - - type: integer - - type: string - description: |- - Borrowed is quantity of quota that is borrowed from the cohort. In other - words, it's the used quota that is over the nominalQuota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - name: - description: name of the resource - type: string - total: - anyOf: - - type: integer - - type: string - description: |- - total is the total quantity of used quota, including the amount borrowed - from the cohort. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - flavorsUsage: - description: |- - flavorsUsage are the used quotas, by flavor, currently in use by the - workloads admitted in this ClusterQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: resources lists the quota usage for the resources - in this flavor. - items: - properties: - borrowed: - anyOf: - - type: integer - - type: string - description: |- - Borrowed is quantity of quota that is borrowed from the cohort. In other - words, it's the used quota that is over the nominalQuota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - name: - description: name of the resource - type: string - total: - anyOf: - - type: integer - - type: string - description: |- - total is the total quantity of used quota, including the amount borrowed - from the cohort. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - pendingWorkloads: - description: |- - pendingWorkloads is the number of workloads currently waiting to be - admitted to this clusterQueue. - format: int32 - type: integer - pendingWorkloadsStatus: - description: |- - PendingWorkloadsStatus contains the information exposed about the current - status of the pending workloads in the cluster queue. - Deprecated: This field will be removed on v1beta2, use VisibilityOnDemand - (https://kueue.sigs.k8s.io/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand/) - instead. - properties: - clusterQueuePendingWorkload: - description: Head contains the list of top pending workloads. - items: - description: |- - ClusterQueuePendingWorkload contains the information identifying a pending workload - in the cluster queue. - properties: - name: - description: Name indicates the name of the pending workload. - type: string - namespace: - description: Namespace indicates the name of the pending - workload. - type: string - required: - - name - - namespace - type: object - type: array - x-kubernetes-list-type: atomic - lastChangeTime: - description: LastChangeTime indicates the time of the last change - of the structure. - format: date-time - type: string - required: - - lastChangeTime - type: object - reservingWorkloads: - description: |- - reservingWorkloads is the number of workloads currently reserving quota in this - clusterQueue. - format: int32 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.17.2 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: cohorts.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: Cohort - listKind: CohortList - plural: cohorts - singular: cohort - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: |- - Cohort defines the Cohorts API. - - Hierarchical Cohorts (any Cohort which has a parent) are compatible - with Fair Sharing as of v0.11. Using these features together in - V0.9 and V0.10 is unsupported, and results in undefined behavior. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: CohortSpec defines the desired state of Cohort - properties: - fairSharing: - description: |- - fairSharing defines the properties of the Cohort when - participating in FairSharing. The values are only relevant - if FairSharing is enabled in the Kueue configuration. - properties: - weight: - anyOf: - - type: integer - - type: string - default: 1 - description: |- - weight gives a comparative advantage to this ClusterQueue - or Cohort when competing for unused resources in the - Cohort. The share is based on the dominant resource usage - above nominal quotas for each resource, divided by the - weight. Admission prioritizes scheduling workloads from - ClusterQueues and Cohorts with the lowest share and - preempting workloads from the ClusterQueues and Cohorts - with the highest share. A zero weight implies infinite - share value, meaning that this Node will always be at - disadvantage against other ClusterQueues and Cohorts. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - parent: - description: |- - Parent references the name of the Cohort's parent, if - any. It satisfies one of three cases: - 1) Unset. This Cohort is the root of its Cohort tree. - 2) References a non-existent Cohort. We use default Cohort (no borrowing/lending limits). - 3) References an existent Cohort. - - If a cycle is created, we disable all members of the - Cohort, including ClusterQueues, until the cycle is - removed. We prevent further admission while the cycle - exists. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resourceGroups: - description: |- - ResourceGroups describes groupings of Resources and - Flavors. Each ResourceGroup defines a list of Resources - and a list of Flavors which provide quotas for these - Resources. Each Resource and each Flavor may only form part - of one ResourceGroup. There may be up to 16 ResourceGroups - within a Cohort. - - BorrowingLimit limits how much members of this Cohort - subtree can borrow from the parent subtree. - - LendingLimit limits how much members of this Cohort subtree - can lend to the parent subtree. - - Borrowing and Lending limits must only be set when the - Cohort has a parent. Otherwise, the Cohort create/update - will be rejected by the webhook. - items: - properties: - coveredResources: - description: |- - coveredResources is the list of resources covered by the flavors in this - group. - Examples: cpu, memory, vendor.com/gpu. - The list cannot be empty and it can contain up to 16 resources. - items: - description: ResourceName is the name identifying various - resources in a ResourceList. - type: string - maxItems: 16 - minItems: 1 - type: array - flavors: - description: |- - flavors is the list of flavors that provide the resources of this group. - Typically, different flavors represent different hardware models - (e.g., gpu models, cpu architectures) or pricing models (on-demand vs spot - cpus). - Each flavor MUST list all the resources listed for this group in the same - order as the .resources field. - The list cannot be empty and it can contain up to 16 flavors. - items: - properties: - name: - description: |- - name of this flavor. The name should match the .metadata.name of a - ResourceFlavor. If a matching ResourceFlavor does not exist, the - ClusterQueue will have an Active condition set to False. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: |- - resources is the list of quotas for this flavor per resource. - There could be up to 16 resources. - items: - properties: - borrowingLimit: - anyOf: - - type: integer - - type: string - description: |- - borrowingLimit is the maximum amount of quota for the [flavor, resource] - combination that this ClusterQueue is allowed to borrow from the unused - quota of other ClusterQueues in the same cohort. - In total, at a given time, Workloads in a ClusterQueue can consume a - quantity of quota equal to nominalQuota+borrowingLimit, assuming the other - ClusterQueues in the cohort have enough unused quota. - If null, it means that there is no borrowing limit. - If not null, it must be non-negative. - borrowingLimit must be null if spec.cohort is empty. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - lendingLimit: - anyOf: - - type: integer - - type: string - description: |- - lendingLimit is the maximum amount of unused quota for the [flavor, resource] - combination that this ClusterQueue can lend to other ClusterQueues in the same cohort. - In total, at a given time, ClusterQueue reserves for its exclusive use - a quantity of quota equals to nominalQuota - lendingLimit. - If null, it means that there is no lending limit, meaning that - all the nominalQuota can be borrowed by other clusterQueues in the cohort. - If not null, it must be non-negative. - lendingLimit must be null if spec.cohort is empty. - This field is in beta stage and is enabled by default. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - name: - description: name of this resource. - type: string - nominalQuota: - anyOf: - - type: integer - - type: string - description: |- - nominalQuota is the quantity of this resource that is available for - Workloads admitted by this ClusterQueue at a point in time. - The nominalQuota must be non-negative. - nominalQuota should represent the resources in the cluster available for - running jobs (after discounting resources consumed by system components - and pods not managed by kueue). In an autoscaled cluster, nominalQuota - should account for resources that can be provided by a component such as - Kubernetes cluster-autoscaler. - - If the ClusterQueue belongs to a cohort, the sum of the quotas for each - (flavor, resource) combination defines the maximum quantity that can be - allocated by a ClusterQueue in the cohort. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - - nominalQuota - type: object - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - coveredResources - - flavors - type: object - x-kubernetes-validations: - - message: flavors must have the same number of resources as the - coveredResources - rule: self.flavors.all(x, size(x.resources) == size(self.coveredResources)) - maxItems: 16 - type: array - x-kubernetes-list-type: atomic - type: object - status: - properties: - fairSharing: - description: fairSharing contains the information about the current - status of Fair Sharing. - properties: - weightedShare: - description: |- - WeightedShare represent the maximum of the ratios of usage - above nominal quota to the lendable resources in the - Cohort, among all the resources provided by the Node, and - divided by the weight. If zero, it means that the usage of - the Node is below the nominal quota. If the Node has a - weight of zero and is borrowing, this will return - 9223372036854775807, the maximum possible share value. - format: int64 - type: integer - required: - - weightedShare - type: object - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - controller-gen.kubebuilder.io/version: v0.17.2 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: localqueues.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: LocalQueue - listKind: LocalQueueList - plural: localqueues - shortNames: - - queue - - queues - - lq - singular: localqueue - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Backing ClusterQueue - jsonPath: .spec.clusterQueue - name: ClusterQueue - type: string - - description: Number of pending workloads - jsonPath: .status.pendingWorkloads - name: Pending Workloads - type: integer - - description: Number of admitted workloads that haven't finished yet. - jsonPath: .status.admittedWorkloads - name: Admitted Workloads - type: integer - name: v1beta1 - schema: - openAPIV3Schema: - description: LocalQueue is the Schema for the localQueues API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: LocalQueueSpec defines the desired state of LocalQueue - properties: - clusterQueue: - description: clusterQueue is a reference to a clusterQueue that backs - this localQueue. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - x-kubernetes-validations: - - message: field is immutable - rule: self == oldSelf - stopPolicy: - default: None - description: |- - stopPolicy - if set to a value different from None, the LocalQueue is considered Inactive, - no new reservation being made. - - Depending on its value, its associated workloads will: - - - None - Workloads are admitted - - HoldAndDrain - Admitted workloads are evicted and Reserving workloads will cancel the reservation. - - Hold - Admitted workloads will run to completion and Reserving workloads will cancel the reservation. - enum: - - None - - Hold - - HoldAndDrain - type: string - type: object - status: - description: LocalQueueStatus defines the observed state of LocalQueue - properties: - admittedWorkloads: - description: |- - admittedWorkloads is the number of workloads in this LocalQueue - admitted to a ClusterQueue and that haven't finished yet. - format: int32 - type: integer - conditions: - description: |- - Conditions hold the latest available observations of the LocalQueue - current state. - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - flavorUsage: - description: |- - flavorsUsage are the used quotas, by flavor currently in use by the - workloads assigned to this LocalQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: resources lists the quota usage for the resources - in this flavor. - items: - properties: - name: - description: name of the resource. - type: string - total: - anyOf: - - type: integer - - type: string - description: total is the total quantity of used quota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - flavors: - description: flavors lists all currently available ResourceFlavors - in specified ClusterQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - nodeLabels: - additionalProperties: - type: string - description: |- - nodeLabels are labels that associate the ResourceFlavor with Nodes that - have the same labels. - maxProperties: 8 - type: object - x-kubernetes-map-type: atomic - nodeTaints: - description: |- - nodeTaints are taints that the nodes associated with this ResourceFlavor - have. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Required. The taint key to be applied to - a node. - type: string - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint - key. - type: string - required: - - effect - - key - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - resources: - description: resources used in the flavor. - items: - description: ResourceName is the name identifying various - resources in a ResourceList. - type: string - maxItems: 16 - type: array - x-kubernetes-list-type: set - topology: - description: |- - topology is the topology that associated with this ResourceFlavor. - - This is an alpha field and requires enabling the TopologyAwareScheduling - feature gate. - properties: - levels: - description: levels define the levels of topology. - items: - type: string - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - name: - description: name is the name of the topology. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - levels - - name - type: object - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - flavorsReservation: - description: |- - flavorsReservation are the reserved quotas, by flavor currently in use by the - workloads assigned to this LocalQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: resources lists the quota usage for the resources - in this flavor. - items: - properties: - name: - description: name of the resource. - type: string - total: - anyOf: - - type: integer - - type: string - description: total is the total quantity of used quota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - pendingWorkloads: - description: PendingWorkloads is the number of Workloads in the LocalQueue - not yet admitted to a ClusterQueue - format: int32 - type: integer - reservingWorkloads: - description: |- - reservingWorkloads is the number of workloads in this LocalQueue - reserving quota in a ClusterQueue and that haven't finished yet. - format: int32 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.17.2 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: multikueueclusters.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: MultiKueueCluster - listKind: MultiKueueClusterList - plural: multikueueclusters - singular: multikueuecluster - scope: Cluster - versions: - - additionalPrinterColumns: - - description: MultiKueueCluster is connected - jsonPath: .status.conditions[?(@.type=='Active')].status - name: Connected - type: string - - description: Time this workload was created - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: MultiKueueCluster is the Schema for the multikueue API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - properties: - kubeConfig: - description: Information how to connect to the cluster. - properties: - location: - description: |- - Location of the KubeConfig. - - If LocationType is Secret then Location is the name of the secret inside the namespace in - which the kueue controller manager is running. The config should be stored in the "kubeconfig" key. - type: string - locationType: - default: Secret - description: Type of the KubeConfig location. - enum: - - Secret - - Path - type: string - required: - - location - - locationType - type: object - required: - - kubeConfig - type: object - status: - properties: - conditions: - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.17.2 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: multikueueconfigs.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: MultiKueueConfig - listKind: MultiKueueConfigList - plural: multikueueconfigs - singular: multikueueconfig - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: MultiKueueConfig is the Schema for the multikueue API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: MultiKueueConfigSpec defines the desired state of MultiKueueConfig - properties: - clusters: - description: List of MultiKueueClusters names where the workloads - from the ClusterQueue should be distributed. - items: - type: string - maxItems: 10 - minItems: 1 - type: array - x-kubernetes-list-type: set - required: - - clusters - type: object - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.17.2 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: provisioningrequestconfigs.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: ProvisioningRequestConfig - listKind: ProvisioningRequestConfigList - plural: provisioningrequestconfigs - singular: provisioningrequestconfig - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: ProvisioningRequestConfig is the Schema for the provisioningrequestconfig - API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: ProvisioningRequestConfigSpec defines the desired state of - ProvisioningRequestConfig - properties: - managedResources: - description: |- - managedResources contains the list of resources managed by the autoscaling. - - If empty, all resources are considered managed. - - If not empty, the ProvisioningRequest will contain only the podsets that are - requesting at least one of them. - - If none of the workloads podsets is requesting at least a managed resource, - the workload is considered ready. - items: - description: ResourceName is the name identifying various resources - in a ResourceList. - type: string - maxItems: 100 - type: array - x-kubernetes-list-type: set - parameters: - additionalProperties: - description: Parameter is limited to 255 characters. - maxLength: 255 - type: string - description: Parameters contains all other parameters classes may - require. - maxProperties: 100 - type: object - provisioningClassName: - description: |- - ProvisioningClassName describes the different modes of provisioning the resources. - Check autoscaling.x-k8s.io ProvisioningRequestSpec.ProvisioningClassName for details. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - retryStrategy: - default: - backoffBaseSeconds: 60 - backoffLimitCount: 3 - backoffMaxSeconds: 1800 - description: |- - retryStrategy defines strategy for retrying ProvisioningRequest. - If null, then the default configuration is applied with the following parameter values: - backoffLimitCount: 3 - backoffBaseSeconds: 60 - 1 min - backoffMaxSeconds: 1800 - 30 mins - - To switch off retry mechanism - set retryStrategy.backoffLimitCount to 0. - properties: - backoffBaseSeconds: - default: 60 - description: |- - BackoffBaseSeconds defines the base for the exponential backoff for - re-queuing an evicted workload. - - Defaults to 60. - format: int32 - type: integer - backoffLimitCount: - default: 3 - description: |- - BackoffLimitCount defines the maximum number of re-queuing retries. - Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). - - Every backoff duration is about "b*2^(n-1)+Rand" where: - - "b" represents the base set by "BackoffBaseSeconds" parameter, - - "n" represents the "workloadStatus.requeueState.count", - - "Rand" represents the random jitter. - During this time, the workload is taken as an inadmissible and - other workloads will have a chance to be admitted. - By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...). - - Defaults to 3. - format: int32 - type: integer - backoffMaxSeconds: - default: 1800 - description: |- - BackoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload. - - Defaults to 1800. - format: int32 - type: integer - type: object - required: - - provisioningClassName - type: object - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - controller-gen.kubebuilder.io/version: v0.17.2 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: resourceflavors.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: ResourceFlavor - listKind: ResourceFlavorList - plural: resourceflavors - shortNames: - - flavor - - flavors - - rf - singular: resourceflavor - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: ResourceFlavor is the Schema for the resourceflavors API. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: ResourceFlavorSpec defines the desired state of the ResourceFlavor - properties: - nodeLabels: - additionalProperties: - type: string - description: |- - nodeLabels are labels that associate the ResourceFlavor with Nodes that - have the same labels. - When a Workload is admitted, its podsets can only get assigned - ResourceFlavors whose nodeLabels match the nodeSelector and nodeAffinity - fields. - Once a ResourceFlavor is assigned to a podSet, the ResourceFlavor's - nodeLabels should be injected into the pods of the Workload by the - controller that integrates with the Workload object. - - nodeLabels can be up to 8 elements. - maxProperties: 8 - type: object - x-kubernetes-map-type: atomic - nodeTaints: - description: |- - nodeTaints are taints that the nodes associated with this ResourceFlavor - have. - Workloads' podsets must have tolerations for these nodeTaints in order to - get assigned this ResourceFlavor during admission. - Only the 'NoSchedule' and 'NoExecute' taint effects are evaluated, - while 'PreferNoSchedule' is ignored. - - An example of a nodeTaint is - cloud.provider.com/preemptible="true":NoSchedule - - nodeTaints can be up to 8 elements. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Required. The taint key to be applied to a node. - type: string - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - required: - - effect - - key - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - x-kubernetes-validations: - - message: 'supported taint effect values: ''NoSchedule'', ''PreferNoSchedule'', - ''NoExecute''' - rule: self.all(x, x.effect in ['NoSchedule', 'PreferNoSchedule', - 'NoExecute']) - tolerations: - description: |- - tolerations are extra tolerations that will be added to the pods admitted in - the quota associated with this resource flavor. - - An example of a toleration is - cloud.provider.com/preemptible="true":NoSchedule - - tolerations can be up to 8 elements. - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - x-kubernetes-validations: - - message: operator must be Exists when 'key' is empty, which means - 'match all values and all keys' - rule: 'self.all(x, !has(x.key) ? x.operator == ''Exists'' : true)' - - message: effect must be 'NoExecute' when 'tolerationSeconds' is - set - rule: 'self.all(x, has(x.tolerationSeconds) ? x.effect == ''NoExecute'' - : true)' - - message: 'supported toleration values: ''Equal''(default), ''Exists''' - rule: self.all(x, !has(x.operator) || x.operator in ['Equal', 'Exists']) - - message: a value must be empty when 'operator' is 'Exists' - rule: 'self.all(x, has(x.operator) && x.operator == ''Exists'' ? - !has(x.value) : true)' - - message: 'supported taint effect values: ''NoSchedule'', ''PreferNoSchedule'', - ''NoExecute''' - rule: self.all(x, !has(x.effect) || x.effect in ['NoSchedule', 'PreferNoSchedule', - 'NoExecute']) - topologyName: - description: |- - topologyName indicates topology for the TAS ResourceFlavor. - When specified, it enables scraping of the topology information from the - nodes matching to the Resource Flavor node labels. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - type: object - x-kubernetes-validations: - - message: at least one nodeLabel is required when topology is set - rule: '!has(self.topologyName) || self.nodeLabels.size() >= 1' - - message: resourceFlavorSpec are immutable when topologyName is set - rule: '!has(oldSelf.topologyName) || self == oldSelf' - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.17.2 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: topologies.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: Topology - listKind: TopologyList - plural: topologies - singular: topology - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: Topology is the Schema for the topology API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: TopologySpec defines the desired state of Topology - properties: - levels: - description: levels define the levels of topology. - items: - description: TopologyLevel defines the desired state of TopologyLevel - properties: - nodeLabel: - description: |- - nodeLabel indicates the name of the node label for a specific topology - level. - - Examples: - - cloud.provider.com/topology-block - - cloud.provider.com/topology-rack - maxLength: 316 - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - nodeLabel - type: object - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - x-kubernetes-validations: - - message: field is immutable - rule: self == oldSelf - - message: must be unique - rule: size(self.filter(i, size(self.filter(j, j == i)) > 1)) == - 0 - - message: the kubernetes.io/hostname label can only be used at the - lowest level of topology - rule: size(self.filter(i, i.nodeLabel == 'kubernetes.io/hostname')) - == 0 || self[size(self) - 1].nodeLabel == 'kubernetes.io/hostname' - required: - - levels - type: object - required: - - spec - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.17.2 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: workloadpriorityclasses.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: WorkloadPriorityClass - listKind: WorkloadPriorityClassList - plural: workloadpriorityclasses - singular: workloadpriorityclass - scope: Cluster - versions: - - additionalPrinterColumns: - - description: Value of workloadPriorityClass's Priority - jsonPath: .value - name: Value - type: integer - name: v1beta1 - schema: - openAPIV3Schema: - description: WorkloadPriorityClass is the Schema for the workloadPriorityClass - API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - description: - description: |- - description is an arbitrary string that usually provides guidelines on - when this workloadPriorityClass should be used. - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - value: - description: |- - value represents the integer value of this workloadPriorityClass. This is the actual priority that workloads - receive when jobs have the name of this class in their workloadPriorityClass label. - Changing the value of workloadPriorityClass doesn't affect the priority of workloads that were already created. - format: int32 - type: integer - required: - - value - type: object - served: true - storage: true - subresources: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - controller-gen.kubebuilder.io/version: v0.17.2 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: workloads.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: Workload - listKind: WorkloadList - plural: workloads - shortNames: - - wl - singular: workload - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Name of the queue this workload was submitted to - jsonPath: .spec.queueName - name: Queue - type: string - - description: Name of the ClusterQueue where the workload is reserving quota - jsonPath: .status.admission.clusterQueue - name: Reserved in - type: string - - description: Admission status - jsonPath: .status.conditions[?(@.type=='Admitted')].status - name: Admitted - type: string - - description: Workload finished - jsonPath: .status.conditions[?(@.type=='Finished')].status - name: Finished - type: string - - description: Time this workload was created - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: Workload is the Schema for the workloads API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: WorkloadSpec defines the desired state of Workload - properties: - active: - default: true - description: |- - Active determines if a workload can be admitted into a queue. - Changing active from true to false will evict any running workloads. - Possible values are: - - - false: indicates that a workload should never be admitted and evicts running workloads - - true: indicates that a workload can be evaluated for admission into it's respective queue. - - Defaults to true - type: boolean - maximumExecutionTimeSeconds: - description: |- - maximumExecutionTimeSeconds if provided, determines the maximum time, in seconds, - the workload can be admitted before it's automatically deactivated. - - If unspecified, no execution time limit is enforced on the Workload. - format: int32 - minimum: 1 - type: integer - podSets: - description: |- - podSets is a list of sets of homogeneous pods, each described by a Pod spec - and a count. - There must be at least one element and at most 8. - podSets cannot be changed. - items: - properties: - count: - default: 1 - description: count is the number of pods for the spec. - format: int32 - minimum: 0 - type: integer - minCount: - description: |- - minCount is the minimum number of pods for the spec acceptable - if the workload supports partial admission. - - If not provided, partial admission for the current PodSet is not - enabled. - - Only one podSet within the workload can use this. - - This is an alpha field and requires enabling PartialAdmission feature gate. - format: int32 - minimum: 1 - type: integer - name: - default: main - description: name is the PodSet name. - maxLength: 63 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - template: - description: |- - template is the Pod template. - - The only allowed fields in template.metadata are labels and annotations. - - If requests are omitted for a container or initContainer, - they default to the limits if they are explicitly specified for the - container or initContainer. - - During admission, the rules in nodeSelector and - nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution that match - the keys in the nodeLabels from the ResourceFlavors considered for this - Workload are used to filter the ResourceFlavors that can be assigned to - this podSet. - properties: - metadata: - description: |- - Standard object's metadata. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - Specification of the desired behavior of the pod. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - activeDeadlineSeconds: - description: |- - Optional duration in seconds the pod may be active on the node relative to - StartTime before the system will actively try to mark it failed and kill associated containers. - Value must be a positive integer. - format: int64 - type: integer - affinity: - description: If specified, the pod's scheduling constraints - properties: - nodeAffinity: - description: Describes node affinity scheduling - rules for the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node matches the corresponding matchExpressions; the - node(s) with the highest sum are the most preferred. - items: - description: |- - An empty preferred scheduling term matches all objects with implicit weight 0 - (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated - with the corresponding weight. - properties: - matchExpressions: - description: A list of node selector - requirements by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector - requirements by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - description: Weight associated with matching - the corresponding nodeSelectorTerm, - in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector - terms. The terms are ORed. - items: - description: |- - A null or empty node selector term matches no objects. The requirements of - them are ANDed. - The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector - requirements by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector - requirements by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - description: Describes pod affinity scheduling rules - (e.g. co-locate this pod in the same node, zone, - etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched - WeightedPodAffinityTerm fields are added - per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity - term, associated with the corresponding - weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a - list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a - list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling - rules (e.g. avoid putting this pod in the same - node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the anti-affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched - WeightedPodAffinityTerm fields are added - per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity - term, associated with the corresponding - weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the anti-affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the anti-affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a - list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a - list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - automountServiceAccountToken: - description: AutomountServiceAccountToken indicates - whether a service account token should be automatically - mounted. - type: boolean - containers: - description: |- - List of containers belonging to the pod. - Containers cannot currently be added or removed. - There must be at least one container in a Pod. - Cannot be updated. - items: - description: A single application container that you - want to run within a pod. - properties: - args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment - variable present in a Container. - properties: - name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment - variable's value. Cannot be used if value - is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret - in the pod's namespace - properties: - key: - description: The key of the secret - to select from. Must be a valid - secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source - of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend - to each key in the ConfigMap. Must be - a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command - to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP - GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration - that the container should sleep. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command - to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP - GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration - that the container should sleep. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute - in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection - to a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. - type: string - ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network - port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external - port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute - in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection - to a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents - resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. - type: string - securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label - that applies to the container. - type: string - role: - description: Role is a SELinux role label - that applies to the container. - type: string - type: - description: Type is a SELinux type label - that applies to the container. - type: string - user: - description: User is a SELinux user label - that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is - the name of the GMSA credential spec - to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute - in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection - to a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block - devices to be used by the container. - items: - description: volumeDevice describes a mapping - of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside - of the container that the device will - be mapped to. - type: string - name: - description: name must match the name of - a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. - items: - description: VolumeMount describes a mounting - of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of - a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - dnsConfig: - description: |- - Specifies the DNS parameters of a pod. - Parameters specified here will be merged to the generated DNS - configuration based on DNSPolicy. - properties: - nameservers: - description: |- - A list of DNS name server IP addresses. - This will be appended to the base nameservers generated from DNSPolicy. - Duplicated nameservers will be removed. - items: - type: string - type: array - x-kubernetes-list-type: atomic - options: - description: |- - A list of DNS resolver options. - This will be merged with the base options generated from DNSPolicy. - Duplicated entries will be removed. Resolution options given in Options - will override those that appear in the base DNSPolicy. - items: - description: PodDNSConfigOption defines DNS resolver - options of a pod. - properties: - name: - description: |- - Name is this DNS resolver option's name. - Required. - type: string - value: - description: Value is this DNS resolver option's - value. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - searches: - description: |- - A list of DNS search domains for host-name lookup. - This will be appended to the base search paths generated from DNSPolicy. - Duplicated search paths will be removed. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - dnsPolicy: - description: |- - Set DNS policy for the pod. - Defaults to "ClusterFirst". - Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. - DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. - To have DNS options set along with hostNetwork, you have to specify DNS policy - explicitly to 'ClusterFirstWithHostNet'. - type: string - enableServiceLinks: - description: |- - EnableServiceLinks indicates whether information about services should be injected into pod's - environment variables, matching the syntax of Docker links. - Optional: Defaults to true. - type: boolean - ephemeralContainers: - description: |- - List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing - pod to perform user-initiated actions such as debugging. This list cannot be specified when - creating a pod, and it cannot be modified by updating the pod spec. In order to add an - ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. - items: - description: |- - An EphemeralContainer is a temporary container that you may add to an existing Pod for - user-initiated activities such as debugging. Ephemeral containers have no resource or - scheduling guarantees, and they will not be restarted when they exit or when a Pod is - removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the - Pod to exceed its resource allocation. - - To add an ephemeral container, use the ephemeralcontainers subresource of an existing - Pod. Ephemeral containers may not be removed or restarted. - properties: - args: - description: |- - Arguments to the entrypoint. - The image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment - variable present in a Container. - properties: - name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment - variable's value. Cannot be used if value - is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret - in the pod's namespace - properties: - key: - description: The key of the secret - to select from. Must be a valid - secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source - of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend - to each key in the ConfigMap. Must be - a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: Lifecycle is not allowed for ephemeral - containers. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command - to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP - GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration - that the container should sleep. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command - to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP - GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration - that the container should sleep. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: Probes are not allowed for ephemeral - containers. - properties: - exec: - description: Exec specifies a command to execute - in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection - to a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the ephemeral container specified as a DNS_LABEL. - This name must be unique among all containers, init containers and ephemeral containers. - type: string - ports: - description: Ports are not allowed for ephemeral - containers. - items: - description: ContainerPort represents a network - port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external - port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: Probes are not allowed for ephemeral - containers. - properties: - exec: - description: Exec specifies a command to execute - in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection - to a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents - resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources - already allocated to the pod. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - Restart policy for the container to manage the restart behavior of each - container within a pod. - This may only be set for init containers. You cannot set this field on - ephemeral containers. - type: string - securityContext: - description: |- - Optional: SecurityContext defines the security options the ephemeral container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label - that applies to the container. - type: string - role: - description: Role is a SELinux role label - that applies to the container. - type: string - type: - description: Type is a SELinux type label - that applies to the container. - type: string - user: - description: User is a SELinux user label - that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is - the name of the GMSA credential spec - to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: Probes are not allowed for ephemeral - containers. - properties: - exec: - description: Exec specifies a command to execute - in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection - to a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - targetContainerName: - description: |- - If set, the name of the container from PodSpec that this ephemeral container targets. - The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. - If not set then the ephemeral container uses the namespaces configured in the Pod spec. - - The container runtime must implement support for this feature. If the runtime does not - support namespace targeting then the result of setting this field is undefined. - type: string - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block - devices to be used by the container. - items: - description: volumeDevice describes a mapping - of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside - of the container that the device will - be mapped to. - type: string - name: - description: name must match the name of - a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. - Cannot be updated. - items: - description: VolumeMount describes a mounting - of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of - a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - hostAliases: - description: |- - HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts - file if specified. - items: - description: |- - HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the - pod's hosts file. - properties: - hostnames: - description: Hostnames for the above IP address. - items: - type: string - type: array - x-kubernetes-list-type: atomic - ip: - description: IP address of the host file entry. - type: string - required: - - ip - type: object - type: array - x-kubernetes-list-map-keys: - - ip - x-kubernetes-list-type: map - hostIPC: - description: |- - Use the host's ipc namespace. - Optional: Default to false. - type: boolean - hostNetwork: - description: |- - Host networking requested for this pod. Use the host's network namespace. - If this option is set, the ports that will be used must be specified. - Default to false. - type: boolean - hostPID: - description: |- - Use the host's pid namespace. - Optional: Default to false. - type: boolean - hostUsers: - description: |- - Use the host's user namespace. - Optional: Default to true. - If set to true or not present, the pod will be run in the host user namespace, useful - for when the pod needs a feature only available to the host user namespace, such as - loading a kernel module with CAP_SYS_MODULE. - When set to false, a new userns is created for the pod. Setting false is useful for - mitigating container breakout vulnerabilities even allowing users to run their - containers as root without actually having root privileges on the host. - This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. - type: boolean - hostname: - description: |- - Specifies the hostname of the Pod - If not specified, the pod's hostname will be set to a system-defined value. - type: string - imagePullSecrets: - description: |- - ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. - If specified, these secrets will be passed to individual puller implementations for them to use. - More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod - items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - initContainers: - description: |- - List of initialization containers belonging to the pod. - Init containers are executed in order prior to containers being started. If any - init container fails, the pod is considered to have failed and is handled according - to its restartPolicy. The name for an init container or normal container must be - unique among all containers. - Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. - The resourceRequirements of an init container are taken into account during scheduling - by finding the highest request/limit for each resource type, and then using the max of - of that value or the sum of the normal containers. Limits are applied to init containers - in a similar fashion. - Init containers cannot currently be added or removed. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ - items: - description: A single application container that you - want to run within a pod. - properties: - args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment - variable present in a Container. - properties: - name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment - variable's value. Cannot be used if value - is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret - in the pod's namespace - properties: - key: - description: The key of the secret - to select from. Must be a valid - secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source - of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend - to each key in the ConfigMap. Must be - a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command - to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP - GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration - that the container should sleep. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command - to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP - GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration - that the container should sleep. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute - in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection - to a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. - type: string - ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network - port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external - port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute - in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection - to a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents - resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. - type: string - securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label - that applies to the container. - type: string - role: - description: Role is a SELinux role label - that applies to the container. - type: string - type: - description: Type is a SELinux type label - that applies to the container. - type: string - user: - description: User is a SELinux user label - that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is - the name of the GMSA credential spec - to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute - in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection - to a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block - devices to be used by the container. - items: - description: volumeDevice describes a mapping - of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside - of the container that the device will - be mapped to. - type: string - name: - description: name must match the name of - a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. - items: - description: VolumeMount describes a mounting - of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of - a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - nodeName: - description: |- - NodeName indicates in which node this pod is scheduled. - If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. - Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. - This field should not be used to express a desire for the pod to be scheduled on a specific node. - https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename - type: string - nodeSelector: - additionalProperties: - type: string - description: |- - NodeSelector is a selector which must be true for the pod to fit on a node. - Selector which must match a node's labels for the pod to be scheduled on that node. - More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - type: object - x-kubernetes-map-type: atomic - os: - description: |- - Specifies the OS of the containers in the pod. - Some pod and container fields are restricted if this is set. - - If the OS field is set to linux, the following fields must be unset: - -securityContext.windowsOptions - - If the OS field is set to windows, following fields must be unset: - - spec.hostPID - - spec.hostIPC - - spec.hostUsers - - spec.securityContext.appArmorProfile - - spec.securityContext.seLinuxOptions - - spec.securityContext.seccompProfile - - spec.securityContext.fsGroup - - spec.securityContext.fsGroupChangePolicy - - spec.securityContext.sysctls - - spec.shareProcessNamespace - - spec.securityContext.runAsUser - - spec.securityContext.runAsGroup - - spec.securityContext.supplementalGroups - - spec.securityContext.supplementalGroupsPolicy - - spec.containers[*].securityContext.appArmorProfile - - spec.containers[*].securityContext.seLinuxOptions - - spec.containers[*].securityContext.seccompProfile - - spec.containers[*].securityContext.capabilities - - spec.containers[*].securityContext.readOnlyRootFilesystem - - spec.containers[*].securityContext.privileged - - spec.containers[*].securityContext.allowPrivilegeEscalation - - spec.containers[*].securityContext.procMount - - spec.containers[*].securityContext.runAsUser - - spec.containers[*].securityContext.runAsGroup - properties: - name: - description: |- - Name is the name of the operating system. The currently supported values are linux and windows. - Additional value may be defined in future and can be one of: - https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration - Clients should expect to handle additional values and treat unrecognized values in this field as os: null - type: string - required: - - name - type: object - overhead: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. - This field will be autopopulated at admission time by the RuntimeClass admission controller. If - the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. - The RuntimeClass admission controller will reject Pod create requests which have the overhead already - set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value - defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. - More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md - type: object - preemptionPolicy: - description: |- - PreemptionPolicy is the Policy for preempting pods with lower priority. - One of Never, PreemptLowerPriority. - Defaults to PreemptLowerPriority if unset. - type: string - priority: - description: |- - The priority value. Various system components use this field to find the - priority of the pod. When Priority Admission Controller is enabled, it - prevents users from setting this field. The admission controller populates - this field from PriorityClassName. - The higher the value, the higher the priority. - format: int32 - type: integer - priorityClassName: - description: |- - If specified, indicates the pod's priority. "system-node-critical" and - "system-cluster-critical" are two special keywords which indicate the - highest priorities with the former being the highest priority. Any other - name must be defined by creating a PriorityClass object with that name. - If not specified, the pod priority will be default or zero if there is no - default. - type: string - readinessGates: - description: |- - If specified, all readiness gates will be evaluated for pod readiness. - A pod is ready when all its containers are ready AND - all conditions specified in the readiness gates have status equal to "True" - More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates - items: - description: PodReadinessGate contains the reference - to a pod condition - properties: - conditionType: - description: ConditionType refers to a condition - in the pod's condition list with matching type. - type: string - required: - - conditionType - type: object - type: array - x-kubernetes-list-type: atomic - resourceClaims: - description: |- - ResourceClaims defines which ResourceClaims must be allocated - and reserved before the Pod is allowed to start. The resources - will be made available to those containers which consume them - by name. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. - items: - description: |- - PodResourceClaim references exactly one ResourceClaim, either directly - or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim - for the pod. - - It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. - Containers that need access to the ResourceClaim reference it with this name. - properties: - name: - description: |- - Name uniquely identifies this resource claim inside the pod. - This must be a DNS_LABEL. - type: string - resourceClaimName: - description: |- - ResourceClaimName is the name of a ResourceClaim object in the same - namespace as this pod. - - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. - type: string - resourceClaimTemplateName: - description: |- - ResourceClaimTemplateName is the name of a ResourceClaimTemplate - object in the same namespace as this pod. - - The template will be used to create a new ResourceClaim, which will - be bound to this pod. When this pod is deleted, the ResourceClaim - will also be deleted. The pod name and resource name, along with a - generated component, will be used to form a unique name for the - ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. - - This field is immutable and no changes will be made to the - corresponding ResourceClaim by the control plane after creating the - ResourceClaim. - - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - resources: - description: |- - Resources is the total amount of CPU and Memory resources required by all - containers in the pod. It supports specifying Requests and Limits for - "cpu" and "memory" resource names only. ResourceClaims are not supported. - - This field enables fine-grained control over resource allocation for the - entire pod, allowing resource sharing among containers in a pod. - - This is an alpha field and requires enabling the PodLevelResources feature - gate. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry - in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - Restart policy for all containers within the pod. - One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. - Default to Always. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy - type: string - runtimeClassName: - description: |- - RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used - to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. - If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an - empty definition that uses the default runtime handler. - More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class - type: string - schedulerName: - description: |- - If specified, the pod will be dispatched by specified scheduler. - If not specified, the pod will be dispatched by default scheduler. - type: string - schedulingGates: - description: |- - SchedulingGates is an opaque list of values that if specified will block scheduling the pod. - If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the - scheduler will not attempt to schedule the pod. - - SchedulingGates can only be set at pod creation time, and be removed only afterwards. - items: - description: PodSchedulingGate is associated to a - Pod to guard its scheduling. - properties: - name: - description: |- - Name of the scheduling gate. - Each scheduling gate must have a unique name field. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - securityContext: - description: |- - SecurityContext holds pod-level security attributes and common container settings. - Optional: Defaults to empty. See type description for default values of each field. - properties: - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - fsGroup: - description: |- - A special supplemental group that applies to all containers in a pod. - Some volume types allow the Kubelet to change the ownership of that volume - to be owned by the pod: - - 1. The owning GID will be the FSGroup - 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) - 3. The permission bits are OR'd with rw-rw---- - - If unset, the Kubelet will not modify the ownership and permissions of any volume. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - fsGroupChangePolicy: - description: |- - fsGroupChangePolicy defines behavior of changing ownership and permission of the volume - before being exposed inside Pod. This field will only apply to - volume types which support fsGroup based ownership(and permissions). - It will have no effect on ephemeral volume types such as: secret, configmaps - and emptydir. - Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. - Note that this field cannot be set when spec.os.name is windows. - type: string - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxChangePolicy: - description: |- - seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. - It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. - Valid values are "MountOption" and "Recursive". - - "Recursive" means relabeling of all files on all Pod volumes by the container runtime. - This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. - - "MountOption" mounts all eligible Pod volumes with `-o context` mount option. - This requires all Pods that share the same volume to use the same SELinux label. - It is not possible to share the same volume among privileged and unprivileged Pods. - Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes - whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their - CSIDriver instance. Other volumes are always re-labelled recursively. - "MountOption" value is allowed only when SELinuxMount feature gate is enabled. - - If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. - If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes - and "Recursive" for all other volumes. - - This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. - - All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. - Note that this field cannot be set when spec.os.name is windows. - type: string - seLinuxOptions: - description: |- - The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in SecurityContext. If set in - both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that - applies to the container. - type: string - role: - description: Role is a SELinux role label that - applies to the container. - type: string - type: - description: Type is a SELinux type label that - applies to the container. - type: string - user: - description: User is a SELinux user label that - applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - supplementalGroups: - description: |- - A list of groups applied to the first process run in each container, in - addition to the container's primary GID and fsGroup (if specified). If - the SupplementalGroupsPolicy feature is enabled, the - supplementalGroupsPolicy field determines whether these are in addition - to or instead of any group memberships defined in the container image. - If unspecified, no additional groups are added, though group memberships - defined in the container image may still be used, depending on the - supplementalGroupsPolicy field. - Note that this field cannot be set when spec.os.name is windows. - items: - format: int64 - type: integer - type: array - x-kubernetes-list-type: atomic - supplementalGroupsPolicy: - description: |- - Defines how supplemental groups of the first container processes are calculated. - Valid values are "Merge" and "Strict". If not specified, "Merge" is used. - (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled - and the container runtime must implement support for this feature. - Note that this field cannot be set when spec.os.name is windows. - type: string - sysctls: - description: |- - Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported - sysctls (by the container runtime) might fail to launch. - Note that this field cannot be set when spec.os.name is windows. - items: - description: Sysctl defines a kernel parameter - to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options within a container's SecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name - of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - serviceAccount: - description: |- - DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. - Deprecated: Use serviceAccountName instead. - type: string - serviceAccountName: - description: |- - ServiceAccountName is the name of the ServiceAccount to use to run this pod. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ - type: string - setHostnameAsFQDN: - description: |- - If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). - In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). - In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. - If a pod does not have FQDN, this has no effect. - Default to false. - type: boolean - shareProcessNamespace: - description: |- - Share a single process namespace between all of the containers in a pod. - When this is set containers will be able to view and signal processes from other containers - in the same pod, and the first process in each container will not be assigned PID 1. - HostPID and ShareProcessNamespace cannot both be set. - Optional: Default to false. - type: boolean - subdomain: - description: |- - If specified, the fully qualified Pod hostname will be "...svc.". - If not specified, the pod will not have a domainname at all. - type: string - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - If this value is nil, the default grace period will be used instead. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - Defaults to 30 seconds. - format: int64 - type: integer - tolerations: - description: If specified, the pod's tolerations. - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - topologySpreadConstraints: - description: |- - TopologySpreadConstraints describes how a group of pods ought to spread across topology - domains. Scheduler will schedule pods in a way which abides by the constraints. - All topologySpreadConstraints are ANDed. - items: - description: TopologySpreadConstraint specifies how - to spread matching pods among the given topology. - properties: - labelSelector: - description: |- - LabelSelector is used to find matching pods. - Pods that match this label selector are counted to determine the number of pods - in their corresponding topology domain. - properties: - matchExpressions: - description: matchExpressions is a list of - label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select the pods over which - spreading will be calculated. The keys are used to lookup values from the - incoming pod labels, those key-value labels are ANDed with labelSelector - to select the group of existing pods over which spreading will be calculated - for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - MatchLabelKeys cannot be set when LabelSelector isn't set. - Keys that don't exist in the incoming pod labels will - be ignored. A null or empty list means only match against labelSelector. - - This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxSkew: - description: |- - MaxSkew describes the degree to which pods may be unevenly distributed. - When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference - between the number of matching pods in the target topology and the global minimum. - The global minimum is the minimum number of matching pods in an eligible domain - or zero if the number of eligible domains is less than MinDomains. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 2/2/1: - In this case, the global minimum is 1. - | zone1 | zone2 | zone3 | - | P P | P P | P | - - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; - scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) - violate MaxSkew(1). - - if MaxSkew is 2, incoming pod can be scheduled onto any zone. - When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence - to topologies that satisfy it. - It's a required field. Default value is 1 and 0 is not allowed. - format: int32 - type: integer - minDomains: - description: |- - MinDomains indicates a minimum number of eligible domains. - When the number of eligible domains with matching topology keys is less than minDomains, - Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. - And when the number of eligible domains with matching topology keys equals or greater than minDomains, - this value has no effect on scheduling. - As a result, when the number of eligible domains is less than minDomains, - scheduler won't schedule more than maxSkew Pods to those domains. - If value is nil, the constraint behaves as if MinDomains is equal to 1. - Valid values are integers greater than 0. - When value is not nil, WhenUnsatisfiable must be DoNotSchedule. - - For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same - labelSelector spread as 2/2/2: - | zone1 | zone2 | zone3 | - | P P | P P | P P | - The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. - In this situation, new pod with the same labelSelector cannot be scheduled, - because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, - it will violate MaxSkew. - format: int32 - type: integer - nodeAffinityPolicy: - description: |- - NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector - when calculating pod topology spread skew. Options are: - - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. - - If this value is nil, the behavior is equivalent to the Honor policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. - type: string - nodeTaintsPolicy: - description: |- - NodeTaintsPolicy indicates how we will treat node taints when calculating - pod topology spread skew. Options are: - - Honor: nodes without taints, along with tainted nodes for which the incoming pod - has a toleration, are included. - - Ignore: node taints are ignored. All nodes are included. - - If this value is nil, the behavior is equivalent to the Ignore policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. - type: string - topologyKey: - description: |- - TopologyKey is the key of node labels. Nodes that have a label with this key - and identical values are considered to be in the same topology. - We consider each as a "bucket", and try to put balanced number - of pods into each bucket. - We define a domain as a particular instance of a topology. - Also, we define an eligible domain as a domain whose nodes meet the requirements of - nodeAffinityPolicy and nodeTaintsPolicy. - e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. - And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. - It's a required field. - type: string - whenUnsatisfiable: - description: |- - WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy - the spread constraint. - - DoNotSchedule (default) tells the scheduler not to schedule it. - - ScheduleAnyway tells the scheduler to schedule the pod in any location, - but giving higher precedence to topologies that would help reduce the - skew. - A constraint is considered "Unsatisfiable" for an incoming pod - if and only if every possible node assignment for that pod would violate - "MaxSkew" on some topology. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 3/1/1: - | zone1 | zone2 | zone3 | - | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled - to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies - MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler - won't make it *more* imbalanced. - It's a required field. - type: string - required: - - maxSkew - - topologyKey - - whenUnsatisfiable - type: object - type: array - x-kubernetes-list-map-keys: - - topologyKey - - whenUnsatisfiable - x-kubernetes-list-type: map - volumes: - description: |- - List of volumes that can be mounted by containers belonging to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes - items: - description: Volume represents a named volume in a - pod that may be accessed by any container in the - pod. - properties: - awsElasticBlockStore: - description: |- - awsElasticBlockStore represents an AWS Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - Deprecated: AWSElasticBlockStore is deprecated. All operations for the in-tree - awsElasticBlockStore type are redirected to the ebs.csi.aws.com CSI driver. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - format: int32 - type: integer - readOnly: - description: |- - readOnly value true will force the readOnly setting in VolumeMounts. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: boolean - volumeID: - description: |- - volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: string - required: - - volumeID - type: object - azureDisk: - description: |- - azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. - Deprecated: AzureDisk is deprecated. All operations for the in-tree azureDisk type - are redirected to the disk.csi.azure.com CSI driver. - properties: - cachingMode: - description: 'cachingMode is the Host Caching - mode: None, Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data - disk in the blob storage - type: string - diskURI: - description: diskURI is the URI of data disk - in the blob storage - type: string - fsType: - default: ext4 - description: |- - fsType is Filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - kind: - description: 'kind expected values are Shared: - multiple blob disks per storage account Dedicated: - single blob disk per storage account Managed: - azure managed data disk (only in managed - availability set). defaults to shared' - type: string - readOnly: - default: false - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: |- - azureFile represents an Azure File Service mount on the host and bind mount to the pod. - Deprecated: AzureFile is deprecated. All operations for the in-tree azureFile type - are redirected to the file.csi.azure.com CSI driver. - properties: - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret - that contains Azure Storage Account Name - and Key - type: string - shareName: - description: shareName is the azure share - Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: |- - cephFS represents a Ceph FS mount on the host that shares a pod's lifetime. - Deprecated: CephFS is deprecated and the in-tree cephfs type is no longer supported. - properties: - monitors: - description: |- - monitors is Required: Monitors is a collection of Ceph monitors - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - items: - type: string - type: array - x-kubernetes-list-type: atomic - path: - description: 'path is Optional: Used as the - mounted root, rather than the full Ceph - tree, default is /' - type: string - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: boolean - secretFile: - description: |- - secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - secretRef: - description: |- - secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - user: - description: |- - user is optional: User is the rados user name, default is admin - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - required: - - monitors - type: object - cinder: - description: |- - cinder represents a cinder volume attached and mounted on kubelets host machine. - Deprecated: Cinder is deprecated. All operations for the in-tree cinder type - are redirected to the cinder.csi.openstack.org CSI driver. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: boolean - secretRef: - description: |- - secretRef is optional: points to a secret object containing parameters used to connect - to OpenStack. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - volumeID: - description: |- - volumeID used to identify the volume in cinder. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap - that should populate this volume - properties: - defaultMode: - description: |- - defaultMode is optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path - within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional specify whether the - ConfigMap or its keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - csi: - description: csi (Container Storage Interface) - represents ephemeral storage that is handled - by certain external CSI drivers. - properties: - driver: - description: |- - driver is the name of the CSI driver that handles this volume. - Consult with your admin for the correct name as registered in the cluster. - type: string - fsType: - description: |- - fsType to mount. Ex. "ext4", "xfs", "ntfs". - If not provided, the empty value is passed to the associated CSI driver - which will determine the default filesystem to apply. - type: string - nodePublishSecretRef: - description: |- - nodePublishSecretRef is a reference to the secret object containing - sensitive information to pass to the CSI driver to complete the CSI - NodePublishVolume and NodeUnpublishVolume calls. - This field is optional, and may be empty if no secret is required. If the - secret object contains more than one secret, all secret references are passed. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - readOnly: - description: |- - readOnly specifies a read-only configuration for the volume. - Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: |- - volumeAttributes stores driver-specific properties that are passed to the CSI - driver. Consult your driver's documentation for supported values. - type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API - about the pod that should populate this volume - properties: - defaultMode: - description: |- - Optional: mode bits to use on created files by default. Must be a - Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: Items is a list of downward API - volume file - items: - description: DownwardAPIVolumeFile represents - information to create the file containing - the pod field - properties: - fieldRef: - description: 'Required: Selects a field - of the pod: only annotations, labels, - name, namespace and uid are supported.' - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path is the - relative path name of the file to - be created. Must not be absolute or - contain the ''..'' path. Must be utf-8 - encoded. The first item of the relative - path must not start with ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - emptyDir: - description: |- - emptyDir represents a temporary directory that shares a pod's lifetime. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - properties: - medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: |- - ephemeral represents a volume that is handled by a cluster storage driver. - The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, - and deleted when the pod is removed. - - Use this if: - a) the volume is only needed while the pod runs, - b) features of normal volumes like restoring from snapshot or capacity - tracking are needed, - c) the storage driver is specified through a storage class, and - d) the storage driver supports dynamic volume provisioning through - a PersistentVolumeClaim (see EphemeralVolumeSource for more - information on the connection between this volume type - and PersistentVolumeClaim). - - Use PersistentVolumeClaim or one of the vendor-specific - APIs for volumes that persist for longer than the lifecycle - of an individual pod. - - Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to - be used that way - see the documentation of the driver for - more information. - - A pod can use both types of ephemeral volumes and - persistent volumes at the same time. - properties: - volumeClaimTemplate: - description: |- - Will be used to create a stand-alone PVC to provision the volume. - The pod in which this EphemeralVolumeSource is embedded will be the - owner of the PVC, i.e. the PVC will be deleted together with the - pod. The name of the PVC will be `-` where - `` is the name from the `PodSpec.Volumes` array - entry. Pod validation will reject the pod if the concatenated name - is not valid for a PVC (for example, too long). - - An existing PVC with that name that is not owned by the pod - will *not* be used for the pod to avoid using an unrelated - volume by mistake. Starting the pod is then blocked until - the unrelated PVC is removed. If such a pre-created PVC is - meant to be used by the pod, the PVC has to updated with an - owner reference to the pod once the pod exists. Normally - this should not be necessary, but it may be useful when - manually reconstructing a broken cluster. - - This field is read-only and no changes will be made by Kubernetes - to the PVC after it has been created. - - Required, must not be nil. - properties: - metadata: - description: |- - May contain labels and annotations that will be copied into the PVC - when creating it. No other fields are allowed and will be rejected during - validation. - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - The specification for the PersistentVolumeClaim. The entire content is - copied unchanged into the PVC that gets created from this - template. The same fields as in a PersistentVolumeClaim - are also valid here. - properties: - accessModes: - description: |- - accessModes contains the desired access modes the volume should have. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 - items: - type: string - type: array - x-kubernetes-list-type: atomic - dataSource: - description: |- - dataSource field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - If the provisioner or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified data source. - When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, - and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. - If the namespace is specified, then dataSourceRef will not be copied to dataSource. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type - of resource being referenced - type: string - name: - description: Name is the name - of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: |- - dataSourceRef specifies the object from which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a non-empty API group (non - core object) or a PersistentVolumeClaim object. - When this field is specified, volume binding will only succeed if the type of - the specified object matches some installed volume populator or dynamic - provisioner. - This field will replace the functionality of the dataSource field and as such - if both fields are non-empty, they must have the same value. For backwards - compatibility, when namespace isn't specified in dataSourceRef, - both fields (dataSource and dataSourceRef) will be set to the same - value automatically if one of them is empty and the other is non-empty. - When namespace is specified in dataSourceRef, - dataSource isn't set to the same value and must be empty. - There are three important differences between dataSource and dataSourceRef: - * While dataSource only allows two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping them), dataSourceRef - preserves all values, and generates an error if a disallowed value is - specified. - * While dataSource only allows local objects, dataSourceRef allows objects - in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. - (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type - of resource being referenced - type: string - name: - description: Name is the name - of resource being referenced - type: string - namespace: - description: |- - Namespace is the namespace of resource being referenced - Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. - (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: |- - resources represents the minimum resources the volume should have. - If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements - that are lower than previous value but must still be higher than capacity recorded in the - status field of the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - description: selector is a label query - over volumes to consider for binding. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: |- - storageClassName is the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 - type: string - volumeAttributesClassName: - description: |- - volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. - If specified, the CSI driver will create or update the volume with the attributes defined - in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. - If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be - set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource - exists. - More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). - type: string - volumeMode: - description: |- - volumeMode defines what type of volume is required by the claim. - Value of Filesystem is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding - reference to the PersistentVolume - backing this claim. - type: string - type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource - that is attached to a kubelet's host machine - and then exposed to the pod. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - lun: - description: 'lun is Optional: FC target lun - number' - format: int32 - type: integer - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target - worldwide names (WWNs)' - items: - type: string - type: array - x-kubernetes-list-type: atomic - wwids: - description: |- - wwids Optional: FC volume world wide identifiers (wwids) - Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - flexVolume: - description: |- - flexVolume represents a generic volume resource that is - provisioned/attached using an exec based plugin. - Deprecated: FlexVolume is deprecated. Consider using a CSIDriver instead. - properties: - driver: - description: driver is the name of the driver - to use for this volume. - type: string - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field - holds extra command options if any.' - type: object - readOnly: - description: |- - readOnly is Optional: defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef is Optional: secretRef is reference to the secret object containing - sensitive information to pass to the plugin scripts. This may be - empty if no secret object is specified. If the secret object - contains more than one secret, all secrets are passed to the plugin - scripts. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - required: - - driver - type: object - flocker: - description: |- - flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running. - Deprecated: Flocker is deprecated and the in-tree flocker type is no longer supported. - properties: - datasetName: - description: |- - datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker - should be considered as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the - dataset. This is unique identifier of a - Flocker dataset - type: string - type: object - gcePersistentDisk: - description: |- - gcePersistentDisk represents a GCE Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - Deprecated: GCEPersistentDisk is deprecated. All operations for the in-tree - gcePersistentDisk type are redirected to the pd.csi.storage.gke.io CSI driver. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - properties: - fsType: - description: |- - fsType is filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - format: int32 - type: integer - pdName: - description: |- - pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: boolean - required: - - pdName - type: object - gitRepo: - description: |- - gitRepo represents a git repository at a particular revision. - Deprecated: GitRepo is deprecated. To provision a container with a git repo, mount an - EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir - into the Pod's container. - properties: - directory: - description: |- - directory is the target directory name. - Must not contain or start with '..'. If '.' is supplied, the volume directory will be the - git repository. Otherwise, if specified, the volume will contain the git repository in - the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for - the specified revision. - type: string - required: - - repository - type: object - glusterfs: - description: |- - glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. - Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - More info: https://examples.k8s.io/volumes/glusterfs/README.md - properties: - endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: string - path: - description: |- - path is the Glusterfs volume path. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: string - readOnly: - description: |- - readOnly here will force the Glusterfs volume to be mounted with read-only permissions. - Defaults to false. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: boolean - required: - - endpoints - - path - type: object - hostPath: - description: |- - hostPath represents a pre-existing file or directory on the host - machine that is directly exposed to the container. This is generally - used for system agents or other privileged things that are allowed - to see the host machine. Most containers will NOT need this. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - properties: - path: - description: |- - path of the directory on the host. - If the path is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - type: - description: |- - type for HostPath Volume - Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - required: - - path - type: object - image: - description: |- - image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. - The volume is resolved at pod startup depending on which PullPolicy value is provided: - - - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - - The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. - A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. - The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. - The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. - The volume will be mounted read-only (ro) and non-executable files (noexec). - Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). - The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. - properties: - pullPolicy: - description: |- - Policy for pulling OCI objects. Possible values are: - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - type: string - reference: - description: |- - Required: Image or artifact reference to be used. - Behaves in the same way as pod.spec.containers[*].image. - Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - type: object - iscsi: - description: |- - iscsi represents an ISCSI Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md - properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether - support iSCSI Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether - support iSCSI Session CHAP authentication - type: boolean - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - type: string - initiatorName: - description: |- - initiatorName is the custom iSCSI Initiator Name. - If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface - : will be created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified - Name. - type: string - iscsiInterface: - default: default - description: |- - iscsiInterface is the interface Name that uses an iSCSI transport. - Defaults to 'default' (tcp). - type: string - lun: - description: lun represents iSCSI Target Lun - number. - format: int32 - type: integer - portals: - description: |- - portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - items: - type: string - type: array - x-kubernetes-list-type: atomic - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret - for iSCSI target and initiator authentication - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - targetPortal: - description: |- - targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - description: |- - name of the volume. - Must be a DNS_LABEL and unique within the pod. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - nfs: - description: |- - nfs represents an NFS mount on the host that shares a pod's lifetime - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - properties: - path: - description: |- - path that is exported by the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - readOnly: - description: |- - readOnly here will force the NFS export to be mounted with read-only permissions. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: boolean - server: - description: |- - server is the hostname or IP address of the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: |- - persistentVolumeClaimVolumeSource represents a reference to a - PersistentVolumeClaim in the same namespace. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - properties: - claimName: - description: |- - claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - type: string - readOnly: - description: |- - readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: |- - photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine. - Deprecated: PhotonPersistentDisk is deprecated and the in-tree photonPersistentDisk type is no longer supported. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - pdID: - description: pdID is the ID that identifies - Photon Controller persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: |- - portworxVolume represents a portworx volume attached and mounted on kubelets host machine. - Deprecated: PortworxVolume is deprecated. All operations for the in-tree portworxVolume type - are redirected to the pxd.portworx.com CSI driver when the CSIMigrationPortworx feature-gate - is on. - properties: - fsType: - description: |- - fSType represents the filesystem type to mount - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies - a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources - secrets, configmaps, and downward API - properties: - defaultMode: - description: |- - defaultMode are the mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - sources: - description: |- - sources is the list of volume projections. Each entry in this list - handles one source. - items: - description: |- - Projection that may be projected along with other supported volume types. - Exactly one of these fields must be set. - properties: - clusterTrustBundle: - description: |- - ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field - of ClusterTrustBundle objects in an auto-updating file. - - Alpha, gated by the ClusterTrustBundleProjection feature gate. - - ClusterTrustBundle objects can either be selected by name, or by the - combination of signer name and a label selector. - - Kubelet performs aggressive normalization of the PEM contents written - into the pod filesystem. Esoteric PEM features such as inter-block - comments and block headers are stripped. Certificates are deduplicated. - The ordering of certificates within the file is arbitrary, and Kubelet - may change the order over time. - properties: - labelSelector: - description: |- - Select all ClusterTrustBundles that match this label selector. Only has - effect if signerName is set. Mutually-exclusive with name. If unset, - interpreted as "match nothing". If set but empty, interpreted as "match - everything". - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - name: - description: |- - Select a single ClusterTrustBundle by object name. Mutually-exclusive - with signerName and labelSelector. - type: string - optional: - description: |- - If true, don't block pod startup if the referenced ClusterTrustBundle(s) - aren't available. If using name, then the named ClusterTrustBundle is - allowed not to exist. If using signerName, then the combination of - signerName and labelSelector is allowed to match zero - ClusterTrustBundles. - type: boolean - path: - description: Relative path from - the volume root to write the bundle. - type: string - signerName: - description: |- - Select all ClusterTrustBundles that match this signer name. - Mutually-exclusive with name. The contents of all selected - ClusterTrustBundles will be unified and deduplicated. - type: string - required: - - path - type: object - configMap: - description: configMap information about - the configMap data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key - to a path within a volume. - properties: - key: - description: key is the key - to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional specify whether - the ConfigMap or its keys must - be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - downwardAPI: - description: downwardAPI information - about the downwardAPI data to project - properties: - items: - description: Items is a list of - DownwardAPIVolume file - items: - description: DownwardAPIVolumeFile - represents information to create - the file containing the pod - field - properties: - fieldRef: - description: 'Required: Selects - a field of the pod: only - annotations, labels, name, - namespace and uid are supported.' - properties: - apiVersion: - description: Version of - the schema the FieldPath - is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the - field to select in the - specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path - is the relative path name - of the file to be created. - Must not be absolute or - contain the ''..'' path. - Must be utf-8 encoded. The - first item of the relative - path must not start with - ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container - name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies - the output format of - the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: - resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - secret: - description: secret information about - the secret data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key - to a path within a volume. - properties: - key: - description: key is the key - to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional field specify - whether the Secret or its key - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - serviceAccountToken: - description: serviceAccountToken is - information about the serviceAccountToken - data to project - properties: - audience: - description: |- - audience is the intended audience of the token. A recipient of a token - must identify itself with an identifier specified in the audience of the - token, and otherwise should reject the token. The audience defaults to the - identifier of the apiserver. - type: string - expirationSeconds: - description: |- - expirationSeconds is the requested duration of validity of the service - account token. As the token approaches expiration, the kubelet volume - plugin will proactively rotate the service account token. The kubelet will - start trying to rotate the token if the token is older than 80 percent of - its time to live or if the token is older than 24 hours.Defaults to 1 hour - and must be at least 10 minutes. - format: int64 - type: integer - path: - description: |- - path is the path relative to the mount point of the file to project the - token into. - type: string - required: - - path - type: object - type: object - type: array - x-kubernetes-list-type: atomic - type: object - quobyte: - description: |- - quobyte represents a Quobyte mount on the host that shares a pod's lifetime. - Deprecated: Quobyte is deprecated and the in-tree quobyte type is no longer supported. - properties: - group: - description: |- - group to map volume access to - Default is no group - type: string - readOnly: - description: |- - readOnly here will force the Quobyte volume to be mounted with read-only permissions. - Defaults to false. - type: boolean - registry: - description: |- - registry represents a single or multiple Quobyte Registry services - specified as a string as host:port pair (multiple entries are separated with commas) - which acts as the central registry for volumes - type: string - tenant: - description: |- - tenant owning the given Quobyte volume in the Backend - Used with dynamically provisioned Quobyte volumes, value is set by the plugin - type: string - user: - description: |- - user to map volume access to - Defaults to serivceaccount user - type: string - volume: - description: volume is a string that references - an already created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: |- - rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. - Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - More info: https://examples.k8s.io/volumes/rbd/README.md - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - type: string - image: - description: |- - image is the rados image name. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - keyring: - default: /etc/ceph/keyring - description: |- - keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - monitors: - description: |- - monitors is a collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - items: - type: string - type: array - x-kubernetes-list-type: atomic - pool: - default: rbd - description: |- - pool is the rados pool name. - Default is rbd. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: boolean - secretRef: - description: |- - secretRef is name of the authentication secret for RBDUser. If provided - overrides keyring. - Default is nil. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - user: - default: admin - description: |- - user is the rados user name. - Default is admin. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - required: - - image - - monitors - type: object - scaleIO: - description: |- - scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. - Deprecated: ScaleIO is deprecated and the in-tree scaleIO type is no longer supported. - properties: - fsType: - default: xfs - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". - Default is "xfs". - type: string - gateway: - description: gateway is the host address of - the ScaleIO API Gateway. - type: string - protectionDomain: - description: protectionDomain is the name - of the ScaleIO Protection Domain for the - configured storage. - type: string - readOnly: - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef references to the secret for ScaleIO user and other - sensitive information. If this is not provided, Login operation will fail. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - sslEnabled: - description: sslEnabled Flag enable/disable - SSL communication with Gateway, default - false - type: boolean - storageMode: - default: ThinProvisioned - description: |- - storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage - Pool associated with the protection domain. - type: string - system: - description: system is the name of the storage - system as configured in ScaleIO. - type: string - volumeName: - description: |- - volumeName is the name of a volume already created in the ScaleIO system - that is associated with this volume source. - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - description: |- - secret represents a secret that should populate this volume. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - properties: - defaultMode: - description: |- - defaultMode is Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values - for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items If unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path - within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - optional: - description: optional field specify whether - the Secret or its keys must be defined - type: boolean - secretName: - description: |- - secretName is the name of the secret in the pod's namespace to use. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - type: string - type: object - storageos: - description: |- - storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. - Deprecated: StorageOS is deprecated and the in-tree storageos type is no longer supported. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef specifies the secret to use for obtaining the StorageOS API - credentials. If not specified, default values will be attempted. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - volumeName: - description: |- - volumeName is the human-readable name of the StorageOS volume. Volume - names are only unique within a namespace. - type: string - volumeNamespace: - description: |- - volumeNamespace specifies the scope of the volume within StorageOS. If no - namespace is specified then the Pod's namespace will be used. This allows the - Kubernetes name scoping to be mirrored within StorageOS for tighter integration. - Set VolumeName to any name to override the default behaviour. - Set to "default" if you are not using namespaces within StorageOS. - Namespaces that do not pre-exist within StorageOS will be created. - type: string - type: object - vsphereVolume: - description: |- - vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine. - Deprecated: VsphereVolume is deprecated. All operations for the in-tree vsphereVolume type - are redirected to the csi.vsphere.vmware.com CSI driver. - properties: - fsType: - description: |- - fsType is filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage - Policy Based Management (SPBM) profile ID - associated with the StoragePolicyName. - type: string - storagePolicyName: - description: storagePolicyName is the storage - Policy Based Management (SPBM) profile name. - type: string - volumePath: - description: volumePath is the path that identifies - vSphere volume vmdk - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - containers - type: object - type: object - topologyRequest: - description: topologyRequest defines the topology request for - the PodSet. - properties: - podIndexLabel: - description: |- - PodIndexLabel indicates the name of the label indexing the pods. - For example, in the context of - - kubernetes job this is: kubernetes.io/job-completion-index - - JobSet: kubernetes.io/job-completion-index (inherited from Job) - - Kubeflow: training.kubeflow.org/replica-index - type: string - preferred: - description: |- - preferred indicates the topology level preferred by the PodSet, as - indicated by the `kueue.x-k8s.io/podset-preferred-topology` PodSet - annotation. - type: string - required: - description: |- - required indicates the topology level required by the PodSet, as - indicated by the `kueue.x-k8s.io/podset-required-topology` PodSet - annotation. - type: string - subGroupCount: - description: |- - SubGroupIndexLabel indicates the count of replicated Jobs (groups) within a PodSet. - For example, in the context of JobSet this value is read from jobset.sigs.k8s.io/replicatedjob-replicas. - format: int32 - type: integer - subGroupIndexLabel: - description: |- - SubGroupIndexLabel indicates the name of the label indexing the instances of replicated Jobs (groups) - within a PodSet. For example, in the context of JobSet this is jobset.sigs.k8s.io/job-index. - type: string - unconstrained: - description: |- - unconstrained indicates that Kueue has the freedom to schedule the PodSet within - the entire available capacity, without constraints on the compactness of the placement. - This is indicated by the `kueue.x-k8s.io/podset-unconstrained-topology` PodSet annotation. - type: boolean - type: object - required: - - count - - template - type: object - x-kubernetes-validations: - - message: minCount should be positive and less or equal to count - rule: 'has(self.minCount) ? self.minCount <= self.count : true' - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - priority: - description: |- - Priority determines the order of access to the resources managed by the - ClusterQueue where the workload is queued. - The priority value is populated from PriorityClassName. - The higher the value, the higher the priority. - If priorityClassName is specified, priority must not be null. - format: int32 - type: integer - priorityClassName: - description: |- - If specified, indicates the workload's priority. - "system-node-critical" and "system-cluster-critical" are two special - keywords which indicate the highest priorities with the former being - the highest priority. Any other name must be defined by creating a - PriorityClass object with that name. If not specified, the workload - priority will be default or zero if there is no default. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - priorityClassSource: - default: "" - description: |- - priorityClassSource determines whether the priorityClass field refers to a pod PriorityClass or kueue.x-k8s.io/workloadpriorityclass. - Workload's PriorityClass can accept the name of a pod priorityClass or a workloadPriorityClass. - When using pod PriorityClass, a priorityClassSource field has the scheduling.k8s.io/priorityclass value. - enum: - - kueue.x-k8s.io/workloadpriorityclass - - scheduling.k8s.io/priorityclass - - "" - type: string - queueName: - description: |- - queueName is the name of the LocalQueue the Workload is associated with. - queueName cannot be changed while .status.admission is not null. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - podSets - type: object - x-kubernetes-validations: - - message: priority should not be nil when priorityClassName is set - rule: 'has(self.priorityClassName) ? has(self.priority) : true' - status: - description: WorkloadStatus defines the observed state of Workload - properties: - accumulatedPastExexcutionTimeSeconds: - description: |- - accumulatedPastExexcutionTimeSeconds holds the total time, in seconds, the workload spent - in Admitted state, in the previous `Admit` - `Evict` cycles. - format: int32 - type: integer - admission: - description: |- - admission holds the parameters of the admission of the workload by a - ClusterQueue. admission can be set back to null, but its fields cannot be - changed once set. - properties: - clusterQueue: - description: clusterQueue is the name of the ClusterQueue that - admitted this workload. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - podSetAssignments: - description: PodSetAssignments hold the admission results for - each of the .spec.podSets entries. - items: - properties: - count: - description: |- - count is the number of pods taken into account at admission time. - This field will not change in case of quota reclaim. - Value could be missing for Workloads created before this field was added, - in that case spec.podSets[*].count value will be used. - format: int32 - minimum: 0 - type: integer - flavors: - additionalProperties: - description: ResourceFlavorReference is the name of the - ResourceFlavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - description: Flavors are the flavors assigned to the workload - for each resource. - type: object - name: - default: main - description: Name is the name of the podSet. It should match - one of the names in .spec.podSets. - maxLength: 63 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - resourceUsage: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - resourceUsage keeps track of the total resources all the pods in the podset need to run. - - Beside what is provided in podSet's specs, this calculation takes into account - the LimitRange defaults and RuntimeClass overheads at the moment of admission. - This field will not change in case of quota reclaim. - type: object - topologyAssignment: - description: |- - topologyAssignment indicates the topology assignment divided into - topology domains corresponding to the lowest level of the topology. - The assignment specifies the number of Pods to be scheduled per topology - domain and specifies the node selectors for each topology domain, in the - following way: the node selector keys are specified by the levels field - (same for all domains), and the corresponding node selector value is - specified by the domains.values subfield. If the TopologySpec.Levels field contains - "kubernetes.io/hostname" label, topologyAssignment will contain data only for - this label, and omit higher levels in the topology - - Example: - - topologyAssignment: - levels: - - cloud.provider.com/topology-block - - cloud.provider.com/topology-rack - domains: - - values: [block-1, rack-1] - count: 4 - - values: [block-1, rack-2] - count: 2 - - Here: - - 4 Pods are to be scheduled on nodes matching the node selector: - cloud.provider.com/topology-block: block-1 - cloud.provider.com/topology-rack: rack-1 - - 2 Pods are to be scheduled on nodes matching the node selector: - cloud.provider.com/topology-block: block-1 - cloud.provider.com/topology-rack: rack-2 - - Example: - Below there is an equivalent of the above example assuming, Topology - object defines kubernetes.io/hostname as the lowest level in topology. - Hence we omit higher level of topologies, since the hostname label - is sufficient to explicitly identify a proper node. - - topologyAssignment: - levels: - - kubernetes.io/hostname - domains: - - values: [hostname-1] - count: 4 - - values: [hostname-2] - count: 2 - properties: - domains: - description: |- - domains is a list of topology assignments split by topology domains at - the lowest level of the topology. - items: - properties: - count: - description: |- - count indicates the number of Pods to be scheduled in the topology - domain indicated by the values field. - format: int32 - minimum: 1 - type: integer - values: - description: |- - values is an ordered list of node selector values describing a topology - domain. The values correspond to the consecutive topology levels, from - the highest to the lowest. - items: - type: string - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - required: - - count - - values - type: object - type: array - levels: - description: |- - levels is an ordered list of keys denoting the levels of the assigned - topology (i.e. node label keys), from the highest to the lowest level of - the topology. - items: - type: string - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - required: - - domains - - levels - type: object - required: - - name - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - clusterQueue - - podSetAssignments - type: object - admissionChecks: - description: admissionChecks list all the admission checks required - by the workload and the current status - items: - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - name: - description: name identifies the admission check. - maxLength: 316 - type: string - podSetUpdates: - items: - description: |- - PodSetUpdate contains a list of pod set modifications suggested by AdmissionChecks. - The modifications should be additive only - modifications of already existing keys - or having the same key provided by multiple AdmissionChecks is not allowed and will - result in failure during workload admission. - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - name: - description: Name of the PodSet to modify. Should match - to one of the Workload's PodSets. - maxLength: 63 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - nodeSelector: - additionalProperties: - type: string - type: object - tolerations: - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - maxItems: 8 - type: array - x-kubernetes-validations: - - message: operator must be Exists when 'key' is empty, - which means 'match all values and all keys' - rule: 'self.all(x, !has(x.key) ? x.operator == ''Exists'' - : true)' - - message: effect must be 'NoExecute' when 'tolerationSeconds' - is set - rule: 'self.all(x, has(x.tolerationSeconds) ? x.effect - == ''NoExecute'' : true)' - - message: 'supported toleration values: ''Equal''(default), - ''Exists''' - rule: self.all(x, !has(x.operator) || x.operator in - ['Equal', 'Exists']) - - message: a value must be empty when 'operator' is 'Exists' - rule: 'self.all(x, has(x.operator) && x.operator == - ''Exists'' ? !has(x.value) : true)' - - message: 'supported taint effect values: ''NoSchedule'', - ''PreferNoSchedule'', ''NoExecute''' - rule: self.all(x, !has(x.effect) || x.effect in ['NoSchedule', - 'PreferNoSchedule', 'NoExecute']) - required: - - name - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - state: - description: state of the admissionCheck, one of Pending, Ready, - Retry, Rejected - enum: - - Pending - - Ready - - Retry - - Rejected - type: string - required: - - lastTransitionTime - - message - - name - - state - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - conditions: - description: |- - conditions hold the latest available observations of the Workload - current state. - - The type of the condition could be: - - - Admitted: the Workload was admitted through a ClusterQueue. - - Finished: the associated workload finished running (failed or succeeded). - - PodsReady: at least `.spec.podSets[*].count` Pods are ready or have - succeeded. - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - reclaimablePods: - description: |- - reclaimablePods keeps track of the number pods within a podset for which - the resource reservation is no longer needed. - items: - properties: - count: - description: count is the number of pods for which the requested - resources are no longer needed. - format: int32 - minimum: 0 - type: integer - name: - description: name is the PodSet name. - maxLength: 63 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - count - - name - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - requeueState: - description: |- - requeueState holds the re-queue state - when a workload meets Eviction with PodsReadyTimeout reason. - properties: - count: - description: |- - count records the number of times a workload has been re-queued - When a deactivated (`.spec.activate`=`false`) workload is reactivated (`.spec.activate`=`true`), - this count would be reset to null. - format: int32 - minimum: 0 - type: integer - requeueAt: - description: |- - requeueAt records the time when a workload will be re-queued. - When a deactivated (`.spec.activate`=`false`) workload is reactivated (`.spec.activate`=`true`), - this time would be reset to null. - format: date-time - type: string - type: object - resourceRequests: - description: |- - resourceRequests provides a detailed view of the resources that were - requested by a non-admitted workload when it was considered for admission. - If admission is non-null, resourceRequests will be empty because - admission.resourceUsage contains the detailed information. - items: - properties: - name: - default: main - description: name is the name of the podSet. It should match - one of the names in .spec.podSets. - maxLength: 63 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - resources: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - resources is the total resources all the pods in the podset need to run. - - Beside what is provided in podSet's specs, this value also takes into account - the LimitRange defaults and RuntimeClass overheads at the moment of consideration - and the application of resource.excludeResourcePrefixes and resource.transformations. - type: object - required: - - name - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - type: object - x-kubernetes-validations: - - message: podSetAssignments must have the same number of podSets as the spec - rule: 'has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'') && has(self.status.admission) - ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) - : true' - - message: field is immutable - rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'')) ? (oldSelf.spec.priorityClassSource - == self.spec.priorityClassSource) : true' - - message: field is immutable - rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassName) - && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName - == self.spec.priorityClassName) : true' - - message: field is immutable - rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'')) && (has(self.status) - && has(self.status.conditions) && self.status.conditions.exists(c, c.type - == ''QuotaReserved'' && c.status == ''True'')) && has(oldSelf.spec.queueName) - && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName - : true' - - message: maximumExecutionTimeSeconds is immutable while admitted - rule: ((has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == 'Admitted' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) - && self.status.conditions.exists(c, c.type == 'Admitted' && c.status == - 'True')))?((has(oldSelf.spec.maximumExecutionTimeSeconds)?oldSelf.spec.maximumExecutionTimeSeconds:0) - == (has(self.spec.maximumExecutionTimeSeconds)?self.spec.maximumExecutionTimeSeconds:0)):true - served: true - storage: true - subresources: - status: {} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-controller-manager - namespace: kueue-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-leader-election-role - namespace: kueue-system -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch ---- -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.kueue.x-k8s.io/batch-admin: "true" -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-batch-admin-role ---- -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.kueue.x-k8s.io/batch-user: "true" -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-batch-user-role ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-clusterqueue-editor-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - clusterqueues - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - clusterqueues/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-clusterqueue-viewer-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - clusterqueues - verbs: - - get - - list - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - clusterqueues/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-job-editor-role -rules: -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-job-viewer-role -rules: -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch -- apiGroups: - - batch - resources: - - jobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-jobset-editor-role -rules: -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-jobset-viewer-role -rules: -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets - verbs: - - get - - list - - watch -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-localqueue-editor-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - localqueues - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - localqueues/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-localqueue-viewer-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - localqueues - verbs: - - get - - list - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - localqueues/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-manager-role -rules: -- apiGroups: - - batch - resources: - - cronjobs - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - watch -- apiGroups: - - "" - resources: - - limitranges - - namespaces - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods/finalizers - verbs: - - get - - update -- apiGroups: - - "" - resources: - - pods/status - verbs: - - get - - patch -- apiGroups: - - "" - resources: - - podtemplates - verbs: - - create - - delete - - get - - list - - update - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - update - - watch -- apiGroups: - - admissionregistration.k8s.io - resources: - - mutatingwebhookconfigurations - - validatingwebhookconfigurations - verbs: - - get - - list - - update - - watch -- apiGroups: - - apps - resources: - - deployments - - replicasets - - statefulsets - verbs: - - get - - list - - watch -- apiGroups: - - autoscaling.x-k8s.io - resources: - - provisioningrequests - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - autoscaling.x-k8s.io - resources: - - provisioningrequests/status - verbs: - - get -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs/finalizers - - jobs/status - verbs: - - get - - patch - - update -- apiGroups: - - flowcontrol.apiserver.k8s.io - resources: - - flowschemas - - prioritylevelconfigurations - verbs: - - list - - watch -- apiGroups: - - flowcontrol.apiserver.k8s.io - resources: - - flowschemas/status - verbs: - - patch -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/finalizers - verbs: - - get - - update -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/status - verbs: - - get - - patch - - update -- apiGroups: - - kubeflow.org - resources: - - mpijobs - - paddlejobs - - pytorchjobs - - tfjobs - - xgboostjobs - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - mpijobs/finalizers - - paddlejobs/finalizers - - pytorchjobs/finalizers - - tfjobs/finalizers - - xgboostjobs/finalizers - verbs: - - get - - update -- apiGroups: - - kubeflow.org - resources: - - mpijobs/status - - paddlejobs/status - - pytorchjobs/status - - tfjobs/status - - xgboostjobs/status - verbs: - - get - - patch - - update -- apiGroups: - - kueue.x-k8s.io - resources: - - admissionchecks - - clusterqueues - - cohorts - - localqueues - - workloads - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - admissionchecks/finalizers - - clusterqueues/finalizers - - localqueues/finalizers - - resourceflavors/finalizers - - topologies/finalizers - - workloads/finalizers - verbs: - - update -- apiGroups: - - kueue.x-k8s.io - resources: - - admissionchecks/status - - clusterqueues/status - - cohorts/status - - localqueues/status - - multikueueclusters/status - - workloads/status - verbs: - - get - - patch - - update -- apiGroups: - - kueue.x-k8s.io - resources: - - multikueueclusters - - multikueueconfigs - - provisioningrequestconfigs - - workloadpriorityclasses - verbs: - - get - - list - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - resourceflavors - verbs: - - delete - - get - - list - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - topologies - verbs: - - get - - list - - update - - watch -- apiGroups: - - leaderworkerset.x-k8s.io - resources: - - leaderworkersets - verbs: - - get - - list - - watch -- apiGroups: - - node.k8s.io - resources: - - runtimeclasses - verbs: - - get - - list - - watch -- apiGroups: - - ray.io - resources: - - rayclusters - - rayjobs - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - ray.io - resources: - - rayclusters/finalizers - - rayjobs/finalizers - verbs: - - get - - update -- apiGroups: - - ray.io - resources: - - rayclusters/status - - rayjobs/status - verbs: - - get - - patch - - update -- apiGroups: - - scheduling.k8s.io - resources: - - priorityclasses - verbs: - - get - - list - - watch -- apiGroups: - - workload.codeflare.dev - resources: - - appwrappers - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - workload.codeflare.dev - resources: - - appwrappers/finalizers - verbs: - - get - - update -- apiGroups: - - workload.codeflare.dev - resources: - - appwrappers/status - verbs: - - get - - patch - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-metrics-auth-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-metrics-reader -rules: -- nonResourceURLs: - - /metrics - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-mpijob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - mpijobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - mpijobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-mpijob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - mpijobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - mpijobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-paddlejob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - paddlejobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - paddlejobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-paddlejob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - paddlejobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - paddlejobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-pending-workloads-cq-viewer-role -rules: -- apiGroups: - - visibility.kueue.x-k8s.io - resources: - - clusterqueues/pendingworkloads - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-pending-workloads-lq-viewer-role -rules: -- apiGroups: - - visibility.kueue.x-k8s.io - resources: - - localqueues/pendingworkloads - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-pytorchjob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-pytorchjob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-raycluster-editor-role -rules: -- apiGroups: - - ray.io - resources: - - rayclusters - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - ray.io - resources: - - rayclusters/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-raycluster-viewer-role -rules: -- apiGroups: - - ray.io - resources: - - rayclusters - verbs: - - get - - list - - watch -- apiGroups: - - ray.io - resources: - - rayclusters/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-rayjob-editor-role -rules: -- apiGroups: - - ray.io - resources: - - rayjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - ray.io - resources: - - rayjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-rayjob-viewer-role -rules: -- apiGroups: - - ray.io - resources: - - rayjobs - verbs: - - get - - list - - watch -- apiGroups: - - ray.io - resources: - - rayjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-resourceflavor-editor-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - resourceflavors - verbs: - - create - - delete - - get - - list - - patch - - update - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-resourceflavor-viewer-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - resourceflavors - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-tfjob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - tfjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - tfjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-tfjob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - tfjobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - tfjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-topology-editor-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - topology - verbs: - - create - - delete - - get - - list - - patch - - update - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-topology-viewer-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - topology - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-workload-editor-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - workloads - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - workloads/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-workload-viewer-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - workloads - verbs: - - get - - list - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - workloads/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-xgboostjob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - xgboostjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - xgboostjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-xgboostjob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - xgboostjobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - xgboostjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-visibility-server-auth-reader - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader -subjects: -- kind: ServiceAccount - name: kueue-controller-manager - namespace: kueue-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-leader-election-rolebinding - namespace: kueue-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: kueue-leader-election-role -subjects: -- kind: ServiceAccount - name: kueue-controller-manager - namespace: kueue-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kueue-manager-role -subjects: -- kind: ServiceAccount - name: kueue-controller-manager - namespace: kueue-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-metrics-auth-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kueue-metrics-auth-role -subjects: -- kind: ServiceAccount - name: kueue-controller-manager - namespace: kueue-system ---- -apiVersion: v1 -data: - controller_manager_config.yaml: | - apiVersion: config.kueue.x-k8s.io/v1beta1 - kind: Configuration - health: - healthProbeBindAddress: :8081 - metrics: - bindAddress: :8443 - # enableClusterQueueResources: true - webhook: - port: 9443 - leaderElection: - leaderElect: true - resourceName: c1f6bfd2.kueue.x-k8s.io - controller: - groupKindConcurrency: - Job.batch: 5 - Pod: 5 - Workload.kueue.x-k8s.io: 5 - LocalQueue.kueue.x-k8s.io: 1 - Cohort.kueue.x-k8s.io: 1 - ClusterQueue.kueue.x-k8s.io: 1 - ResourceFlavor.kueue.x-k8s.io: 1 - clientConnection: - qps: 50 - burst: 100 - #pprofBindAddress: :8083 - #waitForPodsReady: - # enable: false - # timeout: 5m - # recoveryTimeout: 3m - # blockAdmission: false - # requeuingStrategy: - # timestamp: Eviction - # backoffLimitCount: null # null indicates infinite requeuing - # backoffBaseSeconds: 60 - # backoffMaxSeconds: 3600 - #manageJobsWithoutQueueName: true - #managedJobsNamespaceSelector: - # matchExpressions: - # - key: kubernetes.io/metadata.name - # operator: NotIn - # values: [ kube-system, kueue-system ] - #internalCertManagement: - # enable: false - # webhookServiceName: "" - # webhookSecretName: "" - integrations: - frameworks: - - "batch/job" - - "kubeflow.org/mpijob" - - "ray.io/rayjob" - - "ray.io/raycluster" - - "jobset.x-k8s.io/jobset" - - "kubeflow.org/paddlejob" - - "kubeflow.org/pytorchjob" - - "kubeflow.org/tfjob" - - "kubeflow.org/xgboostjob" - - "workload.codeflare.dev/appwrapper" - # - "pod" - # - "deployment" # requires enabling pod integration - # - "statefulset" # requires enabling pod integration - # - "leaderworkerset.x-k8s.io/leaderworkerset" # requires enabling pod integration - # externalFrameworks: - # - "Foo.v1.example.com" - #fairSharing: - # enable: true - # preemptionStrategies: [LessThanOrEqualToFinalShare, LessThanInitialShare] - #resources: - # excludeResourcePrefixes: [] - # transformations: - # - input: nvidia.com/mig-4g.5gb - # strategy: Replace | Retain - # outputs: - # example.com/accelerator-memory: 5Gi - # example.com/accelerator-gpc: 4 -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-manager-config - namespace: kueue-system ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-webhook-server-cert - namespace: kueue-system ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-controller-manager-metrics-service - namespace: kueue-system -spec: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: 8443 - selector: - control-plane: controller-manager ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-visibility-server - namespace: kueue-system -spec: - ports: - - name: https - port: 443 - protocol: TCP - targetPort: 8082 - selector: - control-plane: controller-manager ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-webhook-service - namespace: kueue-system -spec: - ports: - - port: 443 - protocol: TCP - targetPort: 9443 - selector: - control-plane: controller-manager ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-controller-manager - namespace: kueue-system -spec: - replicas: 1 - selector: - matchLabels: - control-plane: controller-manager - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: manager - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - spec: - containers: - - args: - - --config=/controller_manager_config.yaml - - --zap-log-level=2 - - --feature-gates=TopologyAwareScheduling=true - command: - - /manager - image: registry.k8s.io/kueue/kueue:v0.11.4 - imagePullPolicy: Always - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - ports: - - containerPort: 8443 - name: metrics - protocol: TCP - - containerPort: 8082 - name: visibility - protocol: TCP - - containerPort: 9443 - name: webhook-server - protocol: TCP - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - cpu: "2" - memory: 512Mi - requests: - cpu: 500m - memory: 512Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true - - mountPath: /controller_manager_config.yaml - name: manager-config - subPath: controller_manager_config.yaml - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - serviceAccountName: kueue-controller-manager - terminationGracePeriodSeconds: 10 - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: kueue-webhook-server-cert - - configMap: - name: kueue-manager-config - name: manager-config - tolerations: - - effect: NoSchedule - key: components.gke.io/gke-managed-components - operator: Equal - value: "true" ---- -apiVersion: apiregistration.k8s.io/v1 -kind: APIService -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: v1beta1.visibility.kueue.x-k8s.io -spec: - group: visibility.kueue.x-k8s.io - groupPriorityMinimum: 100 - insecureSkipTLSVerify: true - service: - name: kueue-visibility-server - namespace: kueue-system - version: v1beta1 - versionPriority: 100 ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-mutating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate--v1-pod - failurePolicy: Fail - name: mpod.kb.io - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kueue-system - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-apps-v1-deployment - failurePolicy: Fail - name: mdeployment.kb.io - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kueue-system - rules: - - apiGroups: - - apps - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - deployments - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-apps-v1-statefulset - failurePolicy: Fail - name: mstatefulset.kb.io - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kueue-system - rules: - - apiGroups: - - apps - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - statefulsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-workload-codeflare-dev-v1beta2-appwrapper - failurePolicy: Fail - name: mappwrapper.kb.io - rules: - - apiGroups: - - workload.codeflare.dev - apiVersions: - - v1beta2 - operations: - - CREATE - resources: - - appwrappers - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-batch-v1-job - failurePolicy: Fail - name: mjob.kb.io - rules: - - apiGroups: - - batch - apiVersions: - - v1 - operations: - - CREATE - resources: - - jobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-jobset-x-k8s-io-v1alpha2-jobset - failurePolicy: Fail - name: mjobset.kb.io - rules: - - apiGroups: - - jobset.x-k8s.io - apiVersions: - - v1alpha2 - operations: - - CREATE - resources: - - jobsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v1-paddlejob - failurePolicy: Fail - name: mpaddlejob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - resources: - - paddlejobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v1-pytorchjob - failurePolicy: Fail - name: mpytorchjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - resources: - - pytorchjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v1-tfjob - failurePolicy: Fail - name: mtfjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - resources: - - tfjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v1-xgboostjob - failurePolicy: Fail - name: mxgboostjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - resources: - - xgboostjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-leaderworkerset-x-k8s-io-v1-leaderworkerset - failurePolicy: Fail - name: mleaderworkerset.kb.io - rules: - - apiGroups: - - leaderworkerset.x-k8s.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - leaderworkersets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v2beta1-mpijob - failurePolicy: Fail - name: mmpijob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v2beta1 - operations: - - CREATE - resources: - - mpijobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-ray-io-v1-raycluster - failurePolicy: Fail - name: mraycluster.kb.io - rules: - - apiGroups: - - ray.io - apiVersions: - - v1 - operations: - - CREATE - resources: - - rayclusters - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-ray-io-v1-rayjob - failurePolicy: Fail - name: mrayjob.kb.io - rules: - - apiGroups: - - ray.io - apiVersions: - - v1 - operations: - - CREATE - resources: - - rayjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kueue-x-k8s-io-v1beta1-clusterqueue - failurePolicy: Fail - name: mclusterqueue.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - resources: - - clusterqueues - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kueue-x-k8s-io-v1beta1-resourceflavor - failurePolicy: Fail - name: mresourceflavor.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - resources: - - resourceflavors - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kueue-x-k8s-io-v1beta1-workload - failurePolicy: Fail - name: mworkload.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - resources: - - workloads - sideEffects: None ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-validating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate--v1-pod - failurePolicy: Fail - name: vpod.kb.io - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kueue-system - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - pods - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-apps-v1-deployment - failurePolicy: Fail - name: vdeployment.kb.io - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kueue-system - rules: - - apiGroups: - - apps - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - deployments - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-apps-v1-statefulset - failurePolicy: Fail - name: vstatefulset.kb.io - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kueue-system - rules: - - apiGroups: - - apps - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - statefulsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-workload-codeflare-dev-v1beta2-appwrapper - failurePolicy: Fail - name: vappwrapper.kb.io - rules: - - apiGroups: - - workload.codeflare.dev - apiVersions: - - v1beta2 - operations: - - CREATE - - UPDATE - resources: - - appwrappers - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-batch-v1-job - failurePolicy: Fail - name: vjob.kb.io - rules: - - apiGroups: - - batch - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - jobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-jobset-x-k8s-io-v1alpha2-jobset - failurePolicy: Fail - name: vjobset.kb.io - rules: - - apiGroups: - - jobset.x-k8s.io - apiVersions: - - v1alpha2 - operations: - - CREATE - - UPDATE - resources: - - jobsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v1-paddlejob - failurePolicy: Fail - name: vpaddlejob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - paddlejobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v1-pytorchjob - failurePolicy: Fail - name: vpytorchjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - pytorchjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v1-tfjob - failurePolicy: Fail - name: vtfjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - tfjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v1-xgboostjob - failurePolicy: Fail - name: vxgboostjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - xgboostjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-leaderworkerset-x-k8s-io-v1-leaderworkerset - failurePolicy: Fail - name: vleaderworkerset.kb.io - rules: - - apiGroups: - - leaderworkerset.x-k8s.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - leaderworkersets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v2beta1-mpijob - failurePolicy: Fail - name: vmpijob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v2beta1 - operations: - - CREATE - - UPDATE - resources: - - mpijobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-ray-io-v1-raycluster - failurePolicy: Fail - name: vraycluster.kb.io - rules: - - apiGroups: - - ray.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - rayclusters - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-ray-io-v1-rayjob - failurePolicy: Fail - name: vrayjob.kb.io - rules: - - apiGroups: - - ray.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - rayjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kueue-x-k8s-io-v1beta1-clusterqueue - failurePolicy: Fail - name: vclusterqueue.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - clusterqueues - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kueue-x-k8s-io-v1alpha1-cohort - failurePolicy: Fail - name: vcohort.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1alpha1 - operations: - - CREATE - - UPDATE - resources: - - cohorts - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kueue-x-k8s-io-v1beta1-resourceflavor - failurePolicy: Fail - name: vresourceflavor.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - resourceflavors - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kueue-x-k8s-io-v1beta1-workload - failurePolicy: Fail - name: vworkload.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - workloads - - workloads/status - sideEffects: None diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.12.2.yaml b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.12.2.yaml deleted file mode 100644 index 3c3f1a1783..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/manifests/kueue-v0.12.2.yaml +++ /dev/null @@ -1,13851 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Namespace -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-system ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.17.3 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: admissionchecks.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: AdmissionCheck - listKind: AdmissionCheckList - plural: admissionchecks - singular: admissioncheck - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: AdmissionCheck is the Schema for the admissionchecks API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: AdmissionCheckSpec defines the desired state of AdmissionCheck - properties: - controllerName: - description: |- - controllerName identifies the controller that processes the AdmissionCheck, - not necessarily a Kubernetes Pod or Deployment name. Cannot be empty. - type: string - x-kubernetes-validations: - - message: field is immutable - rule: self == oldSelf - parameters: - description: |- - Parameters identifies a configuration with additional parameters for the - check. - properties: - apiGroup: - description: ApiGroup is the group for the resource being referenced. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: Kind is the type of the resource being referenced. - maxLength: 63 - pattern: ^(?i)[a-z]([-a-z0-9]*[a-z0-9])?$ - type: string - name: - description: Name is the name of the resource being referenced. - maxLength: 63 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - apiGroup - - kind - - name - type: object - retryDelayMinutes: - default: 15 - description: |- - RetryDelayMinutes specifies how long to keep the workload suspended after - a failed check (after it transitioned to False). When the delay period has passed, the check - state goes to "Unknown". The default is 15 min. - Deprecated: retryDelayMinutes has already been deprecated since v0.8 and will be removed in v1beta2. - format: int64 - type: integer - required: - - controllerName - type: object - status: - description: AdmissionCheckStatus defines the observed state of AdmissionCheck - properties: - conditions: - description: |- - conditions hold the latest available observations of the AdmissionCheck - current state. - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.17.3 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: clusterqueues.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: ClusterQueue - listKind: ClusterQueueList - plural: clusterqueues - shortNames: - - cq - singular: clusterqueue - scope: Cluster - versions: - - additionalPrinterColumns: - - description: Cohort that this ClusterQueue belongs to - jsonPath: .spec.cohort - name: Cohort - type: string - - description: The queueing strategy used to prioritize workloads - jsonPath: .spec.queueingStrategy - name: Strategy - priority: 1 - type: string - - description: Number of pending workloads - jsonPath: .status.pendingWorkloads - name: Pending Workloads - type: integer - - description: Number of admitted workloads that haven't finished yet - jsonPath: .status.admittedWorkloads - name: Admitted Workloads - priority: 1 - type: integer - name: v1beta1 - schema: - openAPIV3Schema: - description: ClusterQueue is the Schema for the clusterQueue API. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: ClusterQueueSpec defines the desired state of ClusterQueue - properties: - admissionChecks: - description: |- - admissionChecks lists the AdmissionChecks required by this ClusterQueue. - Cannot be used along with AdmissionCheckStrategy. - items: - description: AdmissionCheckReference is the name of an AdmissionCheck. - maxLength: 316 - type: string - type: array - admissionChecksStrategy: - description: |- - admissionCheckStrategy defines a list of strategies to determine which ResourceFlavors require AdmissionChecks. - This property cannot be used in conjunction with the 'admissionChecks' property. - properties: - admissionChecks: - description: admissionChecks is a list of strategies for AdmissionChecks - items: - description: AdmissionCheckStrategyRule defines rules for a - single AdmissionCheck - properties: - name: - description: name is an AdmissionCheck's name. - maxLength: 316 - type: string - onFlavors: - description: |- - onFlavors is a list of ResourceFlavors' names that this AdmissionCheck should run for. - If empty, the AdmissionCheck will run for all workloads submitted to the ClusterQueue. - items: - description: ResourceFlavorReference is the name of the - ResourceFlavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - type: array - required: - - name - type: object - type: array - type: object - admissionScope: - description: admissionScope indicates whether ClusterQueue uses the - Admission Fair Sharing - properties: - admissionMode: - description: |- - AdmissionMode indicates which mode for AdmissionFairSharing should be used - in the AdmissionScope. Possible values are: - - UsageBasedAdmissionFairSharing - - NoAdmissionFairSharing - type: string - required: - - admissionMode - type: object - cohort: - description: |- - cohort that this ClusterQueue belongs to. CQs that belong to the - same cohort can borrow unused resources from each other. - - A CQ can be a member of a single borrowing cohort. A workload submitted - to a queue referencing this CQ can borrow quota from any CQ in the cohort. - Only quota for the [resource, flavor] pairs listed in the CQ can be - borrowed. - If empty, this ClusterQueue cannot borrow from any other ClusterQueue and - vice versa. - - A cohort is a name that links CQs together, but it doesn't reference any - object. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - fairSharing: - description: |- - fairSharing defines the properties of the ClusterQueue when - participating in FairSharing. The values are only relevant - if FairSharing is enabled in the Kueue configuration. - properties: - weight: - anyOf: - - type: integer - - type: string - default: 1 - description: |- - weight gives a comparative advantage to this ClusterQueue - or Cohort when competing for unused resources in the - Cohort. The share is based on the dominant resource usage - above nominal quotas for each resource, divided by the - weight. Admission prioritizes scheduling workloads from - ClusterQueues and Cohorts with the lowest share and - preempting workloads from the ClusterQueues and Cohorts - with the highest share. A zero weight implies infinite - share value, meaning that this Node will always be at - disadvantage against other ClusterQueues and Cohorts. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - flavorFungibility: - default: {} - description: |- - flavorFungibility defines whether a workload should try the next flavor - before borrowing or preempting in the flavor being evaluated. - properties: - whenCanBorrow: - default: Borrow - description: |- - whenCanBorrow determines whether a workload should try the next flavor - before borrowing in current flavor. The possible values are: - - - `Borrow` (default): allocate in current flavor if borrowing - is possible. - - `TryNextFlavor`: try next flavor even if the current - flavor has enough resources to borrow. - enum: - - Borrow - - TryNextFlavor - type: string - whenCanPreempt: - default: TryNextFlavor - description: |- - whenCanPreempt determines whether a workload should try the next flavor - before borrowing in current flavor. The possible values are: - - - `Preempt`: allocate in current flavor if it's possible to preempt some workloads. - - `TryNextFlavor` (default): try next flavor even if there are enough - candidates for preemption in the current flavor. - enum: - - Preempt - - TryNextFlavor - type: string - type: object - namespaceSelector: - description: |- - namespaceSelector defines which namespaces are allowed to submit workloads to - this clusterQueue. Beyond this basic support for policy, a policy agent like - Gatekeeper should be used to enforce more advanced policies. - Defaults to null which is a nothing selector (no namespaces eligible). - If set to an empty selector `{}`, then all namespaces are eligible. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - preemption: - default: {} - description: |- - ClusterQueuePreemption contains policies to preempt Workloads from this - ClusterQueue or the ClusterQueue's cohort. - - Preemption may be configured to work in the following scenarios: - - - When a Workload fits within the nominal quota of the ClusterQueue, but - the quota is currently borrowed by other ClusterQueues in the cohort. - We preempt workloads in other ClusterQueues to allow this ClusterQueue to - reclaim its nominal quota. Configured using reclaimWithinCohort. - - When a Workload doesn't fit within the nominal quota of the ClusterQueue - and there are admitted Workloads in the ClusterQueue with lower priority. - Configured using withinClusterQueue. - - When a Workload may fit while both borrowing and preempting - low priority workloads in the Cohort. Configured using borrowWithinCohort. - - When FairSharing is enabled, to maintain fair distribution of - unused resources. See FairSharing documentation. - - The preemption algorithm tries to find a minimal set of Workloads to - preempt to accomomdate the pending Workload, preempting Workloads with - lower priority first. - properties: - borrowWithinCohort: - default: {} - description: |- - BorrowWithinCohort contains configuration which allows to preempt workloads - within cohort while borrowing. It only works with Classical Preemption, - __not__ with Fair Sharing. - properties: - maxPriorityThreshold: - description: |- - maxPriorityThreshold allows to restrict the set of workloads which - might be preempted by a borrowing workload, to only workloads with - priority less than or equal to the specified threshold priority. - When the threshold is not specified, then any workload satisfying the - policy can be preempted by the borrowing workload. - format: int32 - type: integer - policy: - default: Never - description: |- - policy determines the policy for preemption to reclaim quota within cohort while borrowing. - Possible values are: - - `Never` (default): do not allow for preemption, in other - ClusterQueues within the cohort, for a borrowing workload. - - `LowerPriority`: allow preemption, in other ClusterQueues - within the cohort, for a borrowing workload, but only if - the preempted workloads are of lower priority. - enum: - - Never - - LowerPriority - type: string - type: object - reclaimWithinCohort: - default: Never - description: |- - reclaimWithinCohort determines whether a pending Workload can preempt - Workloads from other ClusterQueues in the cohort that are using more than - their nominal quota. The possible values are: - - - `Never` (default): do not preempt Workloads in the cohort. - - `LowerPriority`: **Classic Preemption** if the pending Workload - fits within the nominal quota of its ClusterQueue, only preempt - Workloads in the cohort that have lower priority than the pending - Workload. **Fair Sharing** only preempt Workloads in the cohort that - have lower priority than the pending Workload and that satisfy the - Fair Sharing preemptionStategies. - - `Any`: **Classic Preemption** if the pending Workload fits within - the nominal quota of its ClusterQueue, preempt any Workload in the - cohort, irrespective of priority. **Fair Sharing** preempt Workloads - in the cohort that satisfy the Fair Sharing preemptionStrategies. - enum: - - Never - - LowerPriority - - Any - type: string - withinClusterQueue: - default: Never - description: |- - withinClusterQueue determines whether a pending Workload that doesn't fit - within the nominal quota for its ClusterQueue, can preempt active Workloads in - the ClusterQueue. The possible values are: - - - `Never` (default): do not preempt Workloads in the ClusterQueue. - - `LowerPriority`: only preempt Workloads in the ClusterQueue that have - lower priority than the pending Workload. - - `LowerOrNewerEqualPriority`: only preempt Workloads in the ClusterQueue that - either have a lower priority than the pending workload or equal priority - and are newer than the pending workload. - enum: - - Never - - LowerPriority - - LowerOrNewerEqualPriority - type: string - type: object - x-kubernetes-validations: - - message: reclaimWithinCohort=Never and borrowWithinCohort.Policy!=Never - rule: '!(self.reclaimWithinCohort == ''Never'' && has(self.borrowWithinCohort) - && self.borrowWithinCohort.policy != ''Never'')' - queueingStrategy: - default: BestEffortFIFO - description: |- - QueueingStrategy indicates the queueing strategy of the workloads - across the queues in this ClusterQueue. - Current Supported Strategies: - - - StrictFIFO: workloads are ordered strictly by creation time. - Older workloads that can't be admitted will block admitting newer - workloads even if they fit available quota. - - BestEffortFIFO: workloads are ordered by creation time, - however older workloads that can't be admitted will not block - admitting newer workloads that fit existing quota. - enum: - - StrictFIFO - - BestEffortFIFO - type: string - resourceGroups: - description: |- - resourceGroups describes groups of resources. - Each resource group defines the list of resources and a list of flavors - that provide quotas for these resources. - Each resource and each flavor can only form part of one resource group. - resourceGroups can be up to 16. - items: - properties: - coveredResources: - description: |- - coveredResources is the list of resources covered by the flavors in this - group. - Examples: cpu, memory, vendor.com/gpu. - The list cannot be empty and it can contain up to 16 resources. - items: - description: ResourceName is the name identifying various - resources in a ResourceList. - type: string - maxItems: 16 - minItems: 1 - type: array - flavors: - description: |- - flavors is the list of flavors that provide the resources of this group. - Typically, different flavors represent different hardware models - (e.g., gpu models, cpu architectures) or pricing models (on-demand vs spot - cpus). - Each flavor MUST list all the resources listed for this group in the same - order as the .resources field. - The list cannot be empty and it can contain up to 16 flavors. - items: - properties: - name: - description: |- - name of this flavor. The name should match the .metadata.name of a - ResourceFlavor. If a matching ResourceFlavor does not exist, the - ClusterQueue will have an Active condition set to False. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: |- - resources is the list of quotas for this flavor per resource. - There could be up to 16 resources. - items: - properties: - borrowingLimit: - anyOf: - - type: integer - - type: string - description: |- - borrowingLimit is the maximum amount of quota for the [flavor, resource] - combination that this ClusterQueue is allowed to borrow from the unused - quota of other ClusterQueues in the same cohort. - In total, at a given time, Workloads in a ClusterQueue can consume a - quantity of quota equal to nominalQuota+borrowingLimit, assuming the other - ClusterQueues in the cohort have enough unused quota. - If null, it means that there is no borrowing limit. - If not null, it must be non-negative. - borrowingLimit must be null if spec.cohort is empty. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - lendingLimit: - anyOf: - - type: integer - - type: string - description: |- - lendingLimit is the maximum amount of unused quota for the [flavor, resource] - combination that this ClusterQueue can lend to other ClusterQueues in the same cohort. - In total, at a given time, ClusterQueue reserves for its exclusive use - a quantity of quota equals to nominalQuota - lendingLimit. - If null, it means that there is no lending limit, meaning that - all the nominalQuota can be borrowed by other clusterQueues in the cohort. - If not null, it must be non-negative. - lendingLimit must be null if spec.cohort is empty. - This field is in beta stage and is enabled by default. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - name: - description: name of this resource. - type: string - nominalQuota: - anyOf: - - type: integer - - type: string - description: |- - nominalQuota is the quantity of this resource that is available for - Workloads admitted by this ClusterQueue at a point in time. - The nominalQuota must be non-negative. - nominalQuota should represent the resources in the cluster available for - running jobs (after discounting resources consumed by system components - and pods not managed by kueue). In an autoscaled cluster, nominalQuota - should account for resources that can be provided by a component such as - Kubernetes cluster-autoscaler. - - If the ClusterQueue belongs to a cohort, the sum of the quotas for each - (flavor, resource) combination defines the maximum quantity that can be - allocated by a ClusterQueue in the cohort. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - - nominalQuota - type: object - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - coveredResources - - flavors - type: object - x-kubernetes-validations: - - message: flavors must have the same number of resources as the - coveredResources - rule: self.flavors.all(x, size(x.resources) == size(self.coveredResources)) - maxItems: 16 - type: array - x-kubernetes-list-type: atomic - stopPolicy: - default: None - description: |- - stopPolicy - if set to a value different from None, the ClusterQueue is considered Inactive, no new reservation being - made. - - Depending on its value, its associated workloads will: - - - None - Workloads are admitted - - HoldAndDrain - Admitted workloads are evicted and Reserving workloads will cancel the reservation. - - Hold - Admitted workloads will run to completion and Reserving workloads will cancel the reservation. - enum: - - None - - Hold - - HoldAndDrain - type: string - type: object - x-kubernetes-validations: - - message: borrowingLimit must be nil when cohort is empty - rule: '!has(self.cohort) && has(self.resourceGroups) ? self.resourceGroups.all(rg, - rg.flavors.all(f, f.resources.all(r, !has(r.borrowingLimit)))) : true' - status: - description: ClusterQueueStatus defines the observed state of ClusterQueue - properties: - admittedWorkloads: - description: |- - admittedWorkloads is the number of workloads currently admitted to this - clusterQueue and haven't finished yet. - format: int32 - type: integer - conditions: - description: |- - conditions hold the latest available observations of the ClusterQueue - current state. - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - fairSharing: - description: |- - fairSharing contains the current state for this ClusterQueue - when participating in Fair Sharing. - This is recorded only when Fair Sharing is enabled in the Kueue configuration. - properties: - admissionFairSharingStatus: - description: admissionFairSharingStatus represents information - relevant to the Admission Fair Sharing - properties: - consumedResources: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - ConsumedResources represents the aggregated usage of resources over time, - with decaying function applied. - The value is populated if usage consumption functionality is enabled in Kueue config. - type: object - lastUpdate: - description: LastUpdate is the time when share and consumed - resources were updated. - format: date-time - type: string - required: - - consumedResources - - lastUpdate - type: object - weightedShare: - description: |- - WeightedShare represents the maximum of the ratios of usage - above nominal quota to the lendable resources in the - Cohort, among all the resources provided by the Node, and - divided by the weight. If zero, it means that the usage of - the Node is below the nominal quota. If the Node has a - weight of zero and is borrowing, this will return - 9223372036854775807, the maximum possible share value. - format: int64 - type: integer - required: - - weightedShare - type: object - flavorsReservation: - description: |- - flavorsReservation are the reserved quotas, by flavor, currently in use by the - workloads assigned to this ClusterQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: resources lists the quota usage for the resources - in this flavor. - items: - properties: - borrowed: - anyOf: - - type: integer - - type: string - description: |- - Borrowed is quantity of quota that is borrowed from the cohort. In other - words, it's the used quota that is over the nominalQuota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - name: - description: name of the resource - type: string - total: - anyOf: - - type: integer - - type: string - description: |- - total is the total quantity of used quota, including the amount borrowed - from the cohort. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - flavorsUsage: - description: |- - flavorsUsage are the used quotas, by flavor, currently in use by the - workloads admitted in this ClusterQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: resources lists the quota usage for the resources - in this flavor. - items: - properties: - borrowed: - anyOf: - - type: integer - - type: string - description: |- - Borrowed is quantity of quota that is borrowed from the cohort. In other - words, it's the used quota that is over the nominalQuota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - name: - description: name of the resource - type: string - total: - anyOf: - - type: integer - - type: string - description: |- - total is the total quantity of used quota, including the amount borrowed - from the cohort. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - pendingWorkloads: - description: |- - pendingWorkloads is the number of workloads currently waiting to be - admitted to this clusterQueue. - format: int32 - type: integer - pendingWorkloadsStatus: - description: |- - PendingWorkloadsStatus contains the information exposed about the current - status of the pending workloads in the cluster queue. - Deprecated: This field will be removed on v1beta2, use VisibilityOnDemand - (https://kueue.sigs.k8s.io/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand/) - instead. - properties: - clusterQueuePendingWorkload: - description: Head contains the list of top pending workloads. - items: - description: |- - ClusterQueuePendingWorkload contains the information identifying a pending workload - in the cluster queue. - properties: - name: - description: Name indicates the name of the pending workload. - type: string - namespace: - description: Namespace indicates the name of the pending - workload. - type: string - required: - - name - - namespace - type: object - type: array - x-kubernetes-list-type: atomic - lastChangeTime: - description: LastChangeTime indicates the time of the last change - of the structure. - format: date-time - type: string - required: - - lastChangeTime - type: object - reservingWorkloads: - description: |- - reservingWorkloads is the number of workloads currently reserving quota in this - clusterQueue. - format: int32 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.17.3 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: cohorts.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: Cohort - listKind: CohortList - plural: cohorts - singular: cohort - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: |- - Cohort defines the Cohorts API. - - Hierarchical Cohorts (any Cohort which has a parent) are compatible - with Fair Sharing as of v0.11. Using these features together in - V0.9 and V0.10 is unsupported, and results in undefined behavior. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: CohortSpec defines the desired state of Cohort - properties: - fairSharing: - description: |- - fairSharing defines the properties of the Cohort when - participating in FairSharing. The values are only relevant - if FairSharing is enabled in the Kueue configuration. - properties: - weight: - anyOf: - - type: integer - - type: string - default: 1 - description: |- - weight gives a comparative advantage to this ClusterQueue - or Cohort when competing for unused resources in the - Cohort. The share is based on the dominant resource usage - above nominal quotas for each resource, divided by the - weight. Admission prioritizes scheduling workloads from - ClusterQueues and Cohorts with the lowest share and - preempting workloads from the ClusterQueues and Cohorts - with the highest share. A zero weight implies infinite - share value, meaning that this Node will always be at - disadvantage against other ClusterQueues and Cohorts. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - parent: - description: |- - Parent references the name of the Cohort's parent, if - any. It satisfies one of three cases: - 1) Unset. This Cohort is the root of its Cohort tree. - 2) References a non-existent Cohort. We use default Cohort (no borrowing/lending limits). - 3) References an existent Cohort. - - If a cycle is created, we disable all members of the - Cohort, including ClusterQueues, until the cycle is - removed. We prevent further admission while the cycle - exists. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resourceGroups: - description: |- - ResourceGroups describes groupings of Resources and - Flavors. Each ResourceGroup defines a list of Resources - and a list of Flavors which provide quotas for these - Resources. Each Resource and each Flavor may only form part - of one ResourceGroup. There may be up to 16 ResourceGroups - within a Cohort. - - BorrowingLimit limits how much members of this Cohort - subtree can borrow from the parent subtree. - - LendingLimit limits how much members of this Cohort subtree - can lend to the parent subtree. - - Borrowing and Lending limits must only be set when the - Cohort has a parent. Otherwise, the Cohort create/update - will be rejected by the webhook. - items: - properties: - coveredResources: - description: |- - coveredResources is the list of resources covered by the flavors in this - group. - Examples: cpu, memory, vendor.com/gpu. - The list cannot be empty and it can contain up to 16 resources. - items: - description: ResourceName is the name identifying various - resources in a ResourceList. - type: string - maxItems: 16 - minItems: 1 - type: array - flavors: - description: |- - flavors is the list of flavors that provide the resources of this group. - Typically, different flavors represent different hardware models - (e.g., gpu models, cpu architectures) or pricing models (on-demand vs spot - cpus). - Each flavor MUST list all the resources listed for this group in the same - order as the .resources field. - The list cannot be empty and it can contain up to 16 flavors. - items: - properties: - name: - description: |- - name of this flavor. The name should match the .metadata.name of a - ResourceFlavor. If a matching ResourceFlavor does not exist, the - ClusterQueue will have an Active condition set to False. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: |- - resources is the list of quotas for this flavor per resource. - There could be up to 16 resources. - items: - properties: - borrowingLimit: - anyOf: - - type: integer - - type: string - description: |- - borrowingLimit is the maximum amount of quota for the [flavor, resource] - combination that this ClusterQueue is allowed to borrow from the unused - quota of other ClusterQueues in the same cohort. - In total, at a given time, Workloads in a ClusterQueue can consume a - quantity of quota equal to nominalQuota+borrowingLimit, assuming the other - ClusterQueues in the cohort have enough unused quota. - If null, it means that there is no borrowing limit. - If not null, it must be non-negative. - borrowingLimit must be null if spec.cohort is empty. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - lendingLimit: - anyOf: - - type: integer - - type: string - description: |- - lendingLimit is the maximum amount of unused quota for the [flavor, resource] - combination that this ClusterQueue can lend to other ClusterQueues in the same cohort. - In total, at a given time, ClusterQueue reserves for its exclusive use - a quantity of quota equals to nominalQuota - lendingLimit. - If null, it means that there is no lending limit, meaning that - all the nominalQuota can be borrowed by other clusterQueues in the cohort. - If not null, it must be non-negative. - lendingLimit must be null if spec.cohort is empty. - This field is in beta stage and is enabled by default. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - name: - description: name of this resource. - type: string - nominalQuota: - anyOf: - - type: integer - - type: string - description: |- - nominalQuota is the quantity of this resource that is available for - Workloads admitted by this ClusterQueue at a point in time. - The nominalQuota must be non-negative. - nominalQuota should represent the resources in the cluster available for - running jobs (after discounting resources consumed by system components - and pods not managed by kueue). In an autoscaled cluster, nominalQuota - should account for resources that can be provided by a component such as - Kubernetes cluster-autoscaler. - - If the ClusterQueue belongs to a cohort, the sum of the quotas for each - (flavor, resource) combination defines the maximum quantity that can be - allocated by a ClusterQueue in the cohort. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - - nominalQuota - type: object - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - coveredResources - - flavors - type: object - x-kubernetes-validations: - - message: flavors must have the same number of resources as the - coveredResources - rule: self.flavors.all(x, size(x.resources) == size(self.coveredResources)) - maxItems: 16 - type: array - x-kubernetes-list-type: atomic - type: object - status: - description: CohortStatus defines the observed state of Cohort. - properties: - fairSharing: - description: |- - fairSharing contains the current state for this Cohort - when participating in Fair Sharing. - The is recorded only when Fair Sharing is enabled in the Kueue configuration. - properties: - admissionFairSharingStatus: - description: admissionFairSharingStatus represents information - relevant to the Admission Fair Sharing - properties: - consumedResources: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - ConsumedResources represents the aggregated usage of resources over time, - with decaying function applied. - The value is populated if usage consumption functionality is enabled in Kueue config. - type: object - lastUpdate: - description: LastUpdate is the time when share and consumed - resources were updated. - format: date-time - type: string - required: - - consumedResources - - lastUpdate - type: object - weightedShare: - description: |- - WeightedShare represents the maximum of the ratios of usage - above nominal quota to the lendable resources in the - Cohort, among all the resources provided by the Node, and - divided by the weight. If zero, it means that the usage of - the Node is below the nominal quota. If the Node has a - weight of zero and is borrowing, this will return - 9223372036854775807, the maximum possible share value. - format: int64 - type: integer - required: - - weightedShare - type: object - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.17.3 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: localqueues.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: LocalQueue - listKind: LocalQueueList - plural: localqueues - shortNames: - - queue - - queues - - lq - singular: localqueue - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Backing ClusterQueue - jsonPath: .spec.clusterQueue - name: ClusterQueue - type: string - - description: Number of pending workloads - jsonPath: .status.pendingWorkloads - name: Pending Workloads - type: integer - - description: Number of admitted workloads that haven't finished yet. - jsonPath: .status.admittedWorkloads - name: Admitted Workloads - type: integer - name: v1beta1 - schema: - openAPIV3Schema: - description: LocalQueue is the Schema for the localQueues API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: LocalQueueSpec defines the desired state of LocalQueue - properties: - clusterQueue: - description: clusterQueue is a reference to a clusterQueue that backs - this localQueue. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - x-kubernetes-validations: - - message: field is immutable - rule: self == oldSelf - fairSharing: - description: |- - fairSharing defines the properties of the LocalQueue when - participating in AdmissionFairSharing. The values are only relevant - if AdmissionFairSharing is enabled in the Kueue configuration. - properties: - weight: - anyOf: - - type: integer - - type: string - default: 1 - description: |- - weight gives a comparative advantage to this ClusterQueue - or Cohort when competing for unused resources in the - Cohort. The share is based on the dominant resource usage - above nominal quotas for each resource, divided by the - weight. Admission prioritizes scheduling workloads from - ClusterQueues and Cohorts with the lowest share and - preempting workloads from the ClusterQueues and Cohorts - with the highest share. A zero weight implies infinite - share value, meaning that this Node will always be at - disadvantage against other ClusterQueues and Cohorts. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - stopPolicy: - default: None - description: |- - stopPolicy - if set to a value different from None, the LocalQueue is considered Inactive, - no new reservation being made. - - Depending on its value, its associated workloads will: - - - None - Workloads are admitted - - HoldAndDrain - Admitted workloads are evicted and Reserving workloads will cancel the reservation. - - Hold - Admitted workloads will run to completion and Reserving workloads will cancel the reservation. - enum: - - None - - Hold - - HoldAndDrain - type: string - type: object - status: - description: LocalQueueStatus defines the observed state of LocalQueue - properties: - admittedWorkloads: - description: |- - admittedWorkloads is the number of workloads in this LocalQueue - admitted to a ClusterQueue and that haven't finished yet. - format: int32 - type: integer - conditions: - description: |- - Conditions hold the latest available observations of the LocalQueue - current state. - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - fairSharing: - description: FairSharing contains the information about the current - status of fair sharing. - properties: - admissionFairSharingStatus: - description: admissionFairSharingStatus represents information - relevant to the Admission Fair Sharing - properties: - consumedResources: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - ConsumedResources represents the aggregated usage of resources over time, - with decaying function applied. - The value is populated if usage consumption functionality is enabled in Kueue config. - type: object - lastUpdate: - description: LastUpdate is the time when share and consumed - resources were updated. - format: date-time - type: string - required: - - consumedResources - - lastUpdate - type: object - weightedShare: - description: |- - WeightedShare represents the maximum of the ratios of usage - above nominal quota to the lendable resources in the - Cohort, among all the resources provided by the Node, and - divided by the weight. If zero, it means that the usage of - the Node is below the nominal quota. If the Node has a - weight of zero and is borrowing, this will return - 9223372036854775807, the maximum possible share value. - format: int64 - type: integer - required: - - weightedShare - type: object - flavorUsage: - description: |- - flavorsUsage are the used quotas, by flavor currently in use by the - workloads assigned to this LocalQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: resources lists the quota usage for the resources - in this flavor. - items: - properties: - name: - description: name of the resource. - type: string - total: - anyOf: - - type: integer - - type: string - description: total is the total quantity of used quota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - flavors: - description: flavors lists all currently available ResourceFlavors - in specified ClusterQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - nodeLabels: - additionalProperties: - type: string - description: |- - nodeLabels are labels that associate the ResourceFlavor with Nodes that - have the same labels. - maxProperties: 8 - type: object - x-kubernetes-map-type: atomic - nodeTaints: - description: |- - nodeTaints are taints that the nodes associated with this ResourceFlavor - have. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Required. The taint key to be applied to - a node. - type: string - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint - key. - type: string - required: - - effect - - key - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - resources: - description: resources used in the flavor. - items: - description: ResourceName is the name identifying various - resources in a ResourceList. - type: string - maxItems: 16 - type: array - x-kubernetes-list-type: set - topology: - description: |- - topology is the topology that associated with this ResourceFlavor. - - This is an alpha field and requires enabling the TopologyAwareScheduling - feature gate. - properties: - levels: - description: levels define the levels of topology. - items: - type: string - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - name: - description: name is the name of the topology. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - levels - - name - type: object - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - flavorsReservation: - description: |- - flavorsReservation are the reserved quotas, by flavor currently in use by the - workloads assigned to this LocalQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: resources lists the quota usage for the resources - in this flavor. - items: - properties: - name: - description: name of the resource. - type: string - total: - anyOf: - - type: integer - - type: string - description: total is the total quantity of used quota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - pendingWorkloads: - description: PendingWorkloads is the number of Workloads in the LocalQueue - not yet admitted to a ClusterQueue - format: int32 - type: integer - reservingWorkloads: - description: |- - reservingWorkloads is the number of workloads in this LocalQueue - reserving quota in a ClusterQueue and that haven't finished yet. - format: int32 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.17.3 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: multikueueclusters.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: MultiKueueCluster - listKind: MultiKueueClusterList - plural: multikueueclusters - singular: multikueuecluster - scope: Cluster - versions: - - additionalPrinterColumns: - - description: MultiKueueCluster is connected - jsonPath: .status.conditions[?(@.type=='Active')].status - name: Connected - type: string - - description: Time this workload was created - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: MultiKueueCluster is the Schema for the multikueue API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - properties: - kubeConfig: - description: Information how to connect to the cluster. - properties: - location: - description: |- - Location of the KubeConfig. - - If LocationType is Secret then Location is the name of the secret inside the namespace in - which the kueue controller manager is running. The config should be stored in the "kubeconfig" key. - type: string - locationType: - default: Secret - description: Type of the KubeConfig location. - enum: - - Secret - - Path - type: string - required: - - location - - locationType - type: object - required: - - kubeConfig - type: object - status: - properties: - conditions: - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.17.3 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: multikueueconfigs.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: MultiKueueConfig - listKind: MultiKueueConfigList - plural: multikueueconfigs - singular: multikueueconfig - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: MultiKueueConfig is the Schema for the multikueue API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: MultiKueueConfigSpec defines the desired state of MultiKueueConfig - properties: - clusters: - description: List of MultiKueueClusters names where the workloads - from the ClusterQueue should be distributed. - items: - type: string - maxItems: 10 - minItems: 1 - type: array - x-kubernetes-list-type: set - required: - - clusters - type: object - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.17.3 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: provisioningrequestconfigs.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: ProvisioningRequestConfig - listKind: ProvisioningRequestConfigList - plural: provisioningrequestconfigs - singular: provisioningrequestconfig - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: ProvisioningRequestConfig is the Schema for the provisioningrequestconfig - API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: ProvisioningRequestConfigSpec defines the desired state of - ProvisioningRequestConfig - properties: - managedResources: - description: |- - managedResources contains the list of resources managed by the autoscaling. - - If empty, all resources are considered managed. - - If not empty, the ProvisioningRequest will contain only the podsets that are - requesting at least one of them. - - If none of the workloads podsets is requesting at least a managed resource, - the workload is considered ready. - items: - description: ResourceName is the name identifying various resources - in a ResourceList. - type: string - maxItems: 100 - type: array - x-kubernetes-list-type: set - parameters: - additionalProperties: - description: Parameter is limited to 255 characters. - maxLength: 255 - type: string - description: Parameters contains all other parameters classes may - require. - maxProperties: 100 - type: object - podSetMergePolicy: - description: |- - podSetMergePolicy specifies the policy for merging PodSets before being passed - to the cluster autoscaler. - enum: - - IdenticalPodTemplates - - IdenticalWorkloadSchedulingRequirements - type: string - podSetUpdates: - description: |- - podSetUpdates specifies the update of the workload's PodSetUpdates which - are used to target the provisioned nodes. - properties: - nodeSelector: - description: nodeSelector specifies the list of updates for the - NodeSelector. - items: - properties: - key: - description: key specifies the key for the NodeSelector. - maxLength: 317 - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - valueFromProvisioningClassDetail: - description: |- - valueFromProvisioningClassDetail specifies the key of the - ProvisioningRequest.status.provisioningClassDetails from which the value - is used for the update. - maxLength: 32768 - minLength: 1 - type: string - required: - - key - - valueFromProvisioningClassDetail - type: object - maxItems: 8 - type: array - type: object - provisioningClassName: - description: |- - ProvisioningClassName describes the different modes of provisioning the resources. - Check autoscaling.x-k8s.io ProvisioningRequestSpec.ProvisioningClassName for details. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - retryStrategy: - default: - backoffBaseSeconds: 60 - backoffLimitCount: 3 - backoffMaxSeconds: 1800 - description: |- - retryStrategy defines strategy for retrying ProvisioningRequest. - If null, then the default configuration is applied with the following parameter values: - backoffLimitCount: 3 - backoffBaseSeconds: 60 - 1 min - backoffMaxSeconds: 1800 - 30 mins - - To switch off retry mechanism - set retryStrategy.backoffLimitCount to 0. - properties: - backoffBaseSeconds: - default: 60 - description: |- - BackoffBaseSeconds defines the base for the exponential backoff for - re-queuing an evicted workload. - - Defaults to 60. - format: int32 - type: integer - backoffLimitCount: - default: 3 - description: |- - BackoffLimitCount defines the maximum number of re-queuing retries. - Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). - - Every backoff duration is about "b*2^(n-1)+Rand" where: - - "b" represents the base set by "BackoffBaseSeconds" parameter, - - "n" represents the "workloadStatus.requeueState.count", - - "Rand" represents the random jitter. - During this time, the workload is taken as an inadmissible and - other workloads will have a chance to be admitted. - By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...). - - Defaults to 3. - format: int32 - type: integer - backoffMaxSeconds: - default: 1800 - description: |- - BackoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload. - - Defaults to 1800. - format: int32 - type: integer - type: object - required: - - provisioningClassName - type: object - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.17.3 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: resourceflavors.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: ResourceFlavor - listKind: ResourceFlavorList - plural: resourceflavors - shortNames: - - flavor - - flavors - - rf - singular: resourceflavor - scope: Cluster - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: ResourceFlavor is the Schema for the resourceflavors API. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: ResourceFlavorSpec defines the desired state of the ResourceFlavor - properties: - nodeLabels: - additionalProperties: - type: string - description: |- - nodeLabels are labels that associate the ResourceFlavor with Nodes that - have the same labels. - When a Workload is admitted, its podsets can only get assigned - ResourceFlavors whose nodeLabels match the nodeSelector and nodeAffinity - fields. - Once a ResourceFlavor is assigned to a podSet, the ResourceFlavor's - nodeLabels should be injected into the pods of the Workload by the - controller that integrates with the Workload object. - - nodeLabels can be up to 8 elements. - maxProperties: 8 - type: object - x-kubernetes-map-type: atomic - nodeTaints: - description: |- - nodeTaints are taints that the nodes associated with this ResourceFlavor - have. - Workloads' podsets must have tolerations for these nodeTaints in order to - get assigned this ResourceFlavor during admission. - When this ResourceFlavor has also set the matching tolerations (in .spec.tolerations), - then the nodeTaints are not considered during admission. - Only the 'NoSchedule' and 'NoExecute' taint effects are evaluated, - while 'PreferNoSchedule' is ignored. - - An example of a nodeTaint is - cloud.provider.com/preemptible="true":NoSchedule - - nodeTaints can be up to 8 elements. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Required. The taint key to be applied to a node. - type: string - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - required: - - effect - - key - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - x-kubernetes-validations: - - message: 'supported taint effect values: ''NoSchedule'', ''PreferNoSchedule'', - ''NoExecute''' - rule: self.all(x, x.effect in ['NoSchedule', 'PreferNoSchedule', - 'NoExecute']) - tolerations: - description: |- - tolerations are extra tolerations that will be added to the pods admitted in - the quota associated with this resource flavor. - - An example of a toleration is - cloud.provider.com/preemptible="true":NoSchedule - - tolerations can be up to 8 elements. - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - x-kubernetes-validations: - - message: operator must be Exists when 'key' is empty, which means - 'match all values and all keys' - rule: 'self.all(x, !has(x.key) ? x.operator == ''Exists'' : true)' - - message: effect must be 'NoExecute' when 'tolerationSeconds' is - set - rule: 'self.all(x, has(x.tolerationSeconds) ? x.effect == ''NoExecute'' - : true)' - - message: 'supported toleration values: ''Equal''(default), ''Exists''' - rule: self.all(x, !has(x.operator) || x.operator in ['Equal', 'Exists']) - - message: a value must be empty when 'operator' is 'Exists' - rule: 'self.all(x, has(x.operator) && x.operator == ''Exists'' ? - !has(x.value) : true)' - - message: 'supported taint effect values: ''NoSchedule'', ''PreferNoSchedule'', - ''NoExecute''' - rule: self.all(x, !has(x.effect) || x.effect in ['NoSchedule', 'PreferNoSchedule', - 'NoExecute']) - topologyName: - description: |- - topologyName indicates topology for the TAS ResourceFlavor. - When specified, it enables scraping of the topology information from the - nodes matching to the Resource Flavor node labels. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - type: object - x-kubernetes-validations: - - message: at least one nodeLabel is required when topology is set - rule: '!has(self.topologyName) || self.nodeLabels.size() >= 1' - - message: resourceFlavorSpec are immutable when topologyName is set - rule: '!has(oldSelf.topologyName) || self == oldSelf' - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.17.3 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: topologies.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: Topology - listKind: TopologyList - plural: topologies - singular: topology - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: Topology is the Schema for the topology API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: TopologySpec defines the desired state of Topology - properties: - levels: - description: levels define the levels of topology. - items: - description: TopologyLevel defines the desired state of TopologyLevel - properties: - nodeLabel: - description: |- - nodeLabel indicates the name of the node label for a specific topology - level. - - Examples: - - cloud.provider.com/topology-block - - cloud.provider.com/topology-rack - maxLength: 316 - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - nodeLabel - type: object - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - x-kubernetes-validations: - - message: field is immutable - rule: self == oldSelf - - message: must be unique - rule: size(self.filter(i, size(self.filter(j, j == i)) > 1)) == - 0 - - message: the kubernetes.io/hostname label can only be used at the - lowest level of topology - rule: size(self.filter(i, i.nodeLabel == 'kubernetes.io/hostname')) - == 0 || self[size(self) - 1].nodeLabel == 'kubernetes.io/hostname' - required: - - levels - type: object - required: - - spec - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.17.3 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: workloadpriorityclasses.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: WorkloadPriorityClass - listKind: WorkloadPriorityClassList - plural: workloadpriorityclasses - singular: workloadpriorityclass - scope: Cluster - versions: - - additionalPrinterColumns: - - description: Value of workloadPriorityClass's Priority - jsonPath: .value - name: Value - type: integer - name: v1beta1 - schema: - openAPIV3Schema: - description: WorkloadPriorityClass is the Schema for the workloadPriorityClass - API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - description: - description: |- - description is an arbitrary string that usually provides guidelines on - when this workloadPriorityClass should be used. - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - value: - description: |- - value represents the integer value of this workloadPriorityClass. This is the actual priority that workloads - receive when jobs have the name of this class in their workloadPriorityClass label. - Changing the value of workloadPriorityClass doesn't affect the priority of workloads that were already created. - format: int32 - type: integer - required: - - value - type: object - served: true - storage: true - subresources: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.17.3 - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: workloads.kueue.x-k8s.io -spec: - group: kueue.x-k8s.io - names: - kind: Workload - listKind: WorkloadList - plural: workloads - shortNames: - - wl - singular: workload - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Name of the queue this workload was submitted to - jsonPath: .spec.queueName - name: Queue - type: string - - description: Name of the ClusterQueue where the workload is reserving quota - jsonPath: .status.admission.clusterQueue - name: Reserved in - type: string - - description: Admission status - jsonPath: .status.conditions[?(@.type=='Admitted')].status - name: Admitted - type: string - - description: Workload finished - jsonPath: .status.conditions[?(@.type=='Finished')].status - name: Finished - type: string - - description: Time this workload was created - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: Workload is the Schema for the workloads API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: WorkloadSpec defines the desired state of Workload - properties: - active: - default: true - description: |- - Active determines if a workload can be admitted into a queue. - Changing active from true to false will evict any running workloads. - Possible values are: - - - false: indicates that a workload should never be admitted and evicts running workloads - - true: indicates that a workload can be evaluated for admission into it's respective queue. - - Defaults to true - type: boolean - maximumExecutionTimeSeconds: - description: |- - maximumExecutionTimeSeconds if provided, determines the maximum time, in seconds, - the workload can be admitted before it's automatically deactivated. - - If unspecified, no execution time limit is enforced on the Workload. - format: int32 - minimum: 1 - type: integer - podSets: - description: |- - podSets is a list of sets of homogeneous pods, each described by a Pod spec - and a count. - There must be at least one element and at most 8. - podSets cannot be changed. - items: - properties: - count: - default: 1 - description: count is the number of pods for the spec. - format: int32 - minimum: 0 - type: integer - minCount: - description: |- - minCount is the minimum number of pods for the spec acceptable - if the workload supports partial admission. - - If not provided, partial admission for the current PodSet is not - enabled. - - Only one podSet within the workload can use this. - - This is an alpha field and requires enabling PartialAdmission feature gate. - format: int32 - minimum: 1 - type: integer - name: - default: main - description: name is the PodSet name. - maxLength: 63 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - template: - description: |- - template is the Pod template. - - The only allowed fields in template.metadata are labels and annotations. - - If requests are omitted for a container or initContainer, - they default to the limits if they are explicitly specified for the - container or initContainer. - - During admission, the rules in nodeSelector and - nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution that match - the keys in the nodeLabels from the ResourceFlavors considered for this - Workload are used to filter the ResourceFlavors that can be assigned to - this podSet. - properties: - metadata: - description: |- - Standard object's metadata. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - Specification of the desired behavior of the pod. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status - properties: - activeDeadlineSeconds: - description: |- - Optional duration in seconds the pod may be active on the node relative to - StartTime before the system will actively try to mark it failed and kill associated containers. - Value must be a positive integer. - format: int64 - type: integer - affinity: - description: If specified, the pod's scheduling constraints - properties: - nodeAffinity: - description: Describes node affinity scheduling - rules for the pod. - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node matches the corresponding matchExpressions; the - node(s) with the highest sum are the most preferred. - items: - description: |- - An empty preferred scheduling term matches all objects with implicit weight 0 - (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated - with the corresponding weight. - properties: - matchExpressions: - description: A list of node selector - requirements by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector - requirements by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - description: Weight associated with matching - the corresponding nodeSelectorTerm, - in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector - terms. The terms are ORed. - items: - description: |- - A null or empty node selector term matches no objects. The requirements of - them are ANDed. - The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector - requirements by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector - requirements by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - description: Describes pod affinity scheduling rules - (e.g. co-locate this pod in the same node, zone, - etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched - WeightedPodAffinityTerm fields are added - per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity - term, associated with the corresponding - weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a - list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a - list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling - rules (e.g. avoid putting this pod in the same - node, zone, etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the anti-affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched - WeightedPodAffinityTerm fields are added - per-node to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity - term, associated with the corresponding - weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the anti-affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the anti-affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a - list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a - list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: object - automountServiceAccountToken: - description: AutomountServiceAccountToken indicates - whether a service account token should be automatically - mounted. - type: boolean - containers: - description: |- - List of containers belonging to the pod. - Containers cannot currently be added or removed. - There must be at least one container in a Pod. - Cannot be updated. - items: - description: A single application container that you - want to run within a pod. - properties: - args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment - variable present in a Container. - properties: - name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment - variable's value. Cannot be used if value - is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret - in the pod's namespace - properties: - key: - description: The key of the secret - to select from. Must be a valid - secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source - of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend - to each key in the ConfigMap. Must be - a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command - to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP - GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration - that the container should sleep. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command - to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP - GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration - that the container should sleep. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute - in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection - to a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. - type: string - ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network - port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external - port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute - in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection - to a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents - resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. - type: string - securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label - that applies to the container. - type: string - role: - description: Role is a SELinux role label - that applies to the container. - type: string - type: - description: Type is a SELinux type label - that applies to the container. - type: string - user: - description: User is a SELinux user label - that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is - the name of the GMSA credential spec - to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute - in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection - to a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block - devices to be used by the container. - items: - description: volumeDevice describes a mapping - of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside - of the container that the device will - be mapped to. - type: string - name: - description: name must match the name of - a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. - items: - description: VolumeMount describes a mounting - of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of - a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - dnsConfig: - description: |- - Specifies the DNS parameters of a pod. - Parameters specified here will be merged to the generated DNS - configuration based on DNSPolicy. - properties: - nameservers: - description: |- - A list of DNS name server IP addresses. - This will be appended to the base nameservers generated from DNSPolicy. - Duplicated nameservers will be removed. - items: - type: string - type: array - x-kubernetes-list-type: atomic - options: - description: |- - A list of DNS resolver options. - This will be merged with the base options generated from DNSPolicy. - Duplicated entries will be removed. Resolution options given in Options - will override those that appear in the base DNSPolicy. - items: - description: PodDNSConfigOption defines DNS resolver - options of a pod. - properties: - name: - description: |- - Name is this DNS resolver option's name. - Required. - type: string - value: - description: Value is this DNS resolver option's - value. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - searches: - description: |- - A list of DNS search domains for host-name lookup. - This will be appended to the base search paths generated from DNSPolicy. - Duplicated search paths will be removed. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - dnsPolicy: - description: |- - Set DNS policy for the pod. - Defaults to "ClusterFirst". - Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. - DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. - To have DNS options set along with hostNetwork, you have to specify DNS policy - explicitly to 'ClusterFirstWithHostNet'. - type: string - enableServiceLinks: - description: |- - EnableServiceLinks indicates whether information about services should be injected into pod's - environment variables, matching the syntax of Docker links. - Optional: Defaults to true. - type: boolean - ephemeralContainers: - description: |- - List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing - pod to perform user-initiated actions such as debugging. This list cannot be specified when - creating a pod, and it cannot be modified by updating the pod spec. In order to add an - ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. - items: - description: |- - An EphemeralContainer is a temporary container that you may add to an existing Pod for - user-initiated activities such as debugging. Ephemeral containers have no resource or - scheduling guarantees, and they will not be restarted when they exit or when a Pod is - removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the - Pod to exceed its resource allocation. - - To add an ephemeral container, use the ephemeralcontainers subresource of an existing - Pod. Ephemeral containers may not be removed or restarted. - properties: - args: - description: |- - Arguments to the entrypoint. - The image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment - variable present in a Container. - properties: - name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment - variable's value. Cannot be used if value - is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret - in the pod's namespace - properties: - key: - description: The key of the secret - to select from. Must be a valid - secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source - of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend - to each key in the ConfigMap. Must be - a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: Lifecycle is not allowed for ephemeral - containers. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command - to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP - GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration - that the container should sleep. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command - to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP - GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration - that the container should sleep. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: Probes are not allowed for ephemeral - containers. - properties: - exec: - description: Exec specifies a command to execute - in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection - to a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the ephemeral container specified as a DNS_LABEL. - This name must be unique among all containers, init containers and ephemeral containers. - type: string - ports: - description: Ports are not allowed for ephemeral - containers. - items: - description: ContainerPort represents a network - port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external - port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: Probes are not allowed for ephemeral - containers. - properties: - exec: - description: Exec specifies a command to execute - in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection - to a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents - resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources - already allocated to the pod. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - Restart policy for the container to manage the restart behavior of each - container within a pod. - This may only be set for init containers. You cannot set this field on - ephemeral containers. - type: string - securityContext: - description: |- - Optional: SecurityContext defines the security options the ephemeral container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label - that applies to the container. - type: string - role: - description: Role is a SELinux role label - that applies to the container. - type: string - type: - description: Type is a SELinux type label - that applies to the container. - type: string - user: - description: User is a SELinux user label - that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is - the name of the GMSA credential spec - to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: Probes are not allowed for ephemeral - containers. - properties: - exec: - description: Exec specifies a command to execute - in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection - to a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - targetContainerName: - description: |- - If set, the name of the container from PodSpec that this ephemeral container targets. - The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. - If not set then the ephemeral container uses the namespaces configured in the Pod spec. - - The container runtime must implement support for this feature. If the runtime does not - support namespace targeting then the result of setting this field is undefined. - type: string - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block - devices to be used by the container. - items: - description: volumeDevice describes a mapping - of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside - of the container that the device will - be mapped to. - type: string - name: - description: name must match the name of - a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. - Cannot be updated. - items: - description: VolumeMount describes a mounting - of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of - a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - hostAliases: - description: |- - HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts - file if specified. - items: - description: |- - HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the - pod's hosts file. - properties: - hostnames: - description: Hostnames for the above IP address. - items: - type: string - type: array - x-kubernetes-list-type: atomic - ip: - description: IP address of the host file entry. - type: string - required: - - ip - type: object - type: array - x-kubernetes-list-map-keys: - - ip - x-kubernetes-list-type: map - hostIPC: - description: |- - Use the host's ipc namespace. - Optional: Default to false. - type: boolean - hostNetwork: - description: |- - Host networking requested for this pod. Use the host's network namespace. - If this option is set, the ports that will be used must be specified. - Default to false. - type: boolean - hostPID: - description: |- - Use the host's pid namespace. - Optional: Default to false. - type: boolean - hostUsers: - description: |- - Use the host's user namespace. - Optional: Default to true. - If set to true or not present, the pod will be run in the host user namespace, useful - for when the pod needs a feature only available to the host user namespace, such as - loading a kernel module with CAP_SYS_MODULE. - When set to false, a new userns is created for the pod. Setting false is useful for - mitigating container breakout vulnerabilities even allowing users to run their - containers as root without actually having root privileges on the host. - This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. - type: boolean - hostname: - description: |- - Specifies the hostname of the Pod - If not specified, the pod's hostname will be set to a system-defined value. - type: string - imagePullSecrets: - description: |- - ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. - If specified, these secrets will be passed to individual puller implementations for them to use. - More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod - items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - initContainers: - description: |- - List of initialization containers belonging to the pod. - Init containers are executed in order prior to containers being started. If any - init container fails, the pod is considered to have failed and is handled according - to its restartPolicy. The name for an init container or normal container must be - unique among all containers. - Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. - The resourceRequirements of an init container are taken into account during scheduling - by finding the highest request/limit for each resource type, and then using the max of - of that value or the sum of the normal containers. Limits are applied to init containers - in a similar fashion. - Init containers cannot currently be added or removed. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ - items: - description: A single application container that you - want to run within a pod. - properties: - args: - description: |- - Arguments to the entrypoint. - The container image's CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - command: - description: |- - Entrypoint array. Not executed within a shell. - The container image's ENTRYPOINT is used if this is not provided. - Variable references $(VAR_NAME) are expanded using the container's environment. If a variable - cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will - produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless - of whether the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell - items: - type: string - type: array - x-kubernetes-list-type: atomic - env: - description: |- - List of environment variables to set in the container. - Cannot be updated. - items: - description: EnvVar represents an environment - variable present in a Container. - properties: - name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. - type: string - value: - description: |- - Variable references $(VAR_NAME) are expanded - using the previously defined environment variables in the container and - any service environment variables. If a variable cannot be resolved, - the reference in the input string will be unchanged. Double $$ are reduced - to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. - "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". - Escaped references will never be expanded, regardless of whether the variable - exists or not. - Defaults to "". - type: string - valueFrom: - description: Source for the environment - variable's value. Cannot be used if value - is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - ConfigMap or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - fieldRef: - description: |- - Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, - spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. - properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - secretKeyRef: - description: Selects a key of a secret - in the pod's namespace - properties: - key: - description: The key of the secret - to select from. Must be a valid - secret key. - type: string - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the - Secret or its key must be defined - type: boolean - required: - - key - type: object - x-kubernetes-map-type: atomic - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - envFrom: - description: |- - List of sources to populate environment variables in the container. - The keys defined within a source must be a C_IDENTIFIER. All invalid keys - will be reported as an event when the container is starting. When a key exists in multiple - sources, the value associated with the last source will take precedence. - Values defined by an Env with a duplicate key will take precedence. - Cannot be updated. - items: - description: EnvFromSource represents the source - of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the ConfigMap - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - prefix: - description: An optional identifier to prepend - to each key in the ConfigMap. Must be - a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: Specify whether the Secret - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - type: object - type: array - x-kubernetes-list-type: atomic - image: - description: |- - Container image name. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - imagePullPolicy: - description: |- - Image pull policy. - One of Always, Never, IfNotPresent. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/containers/images#updating-images - type: string - lifecycle: - description: |- - Actions that the management system should take in response to container lifecycle events. - Cannot be updated. - properties: - postStart: - description: |- - PostStart is called immediately after a container is created. If the handler fails, - the container is terminated and restarted according to its restart policy. - Other management of the container blocks until the hook completes. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command - to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP - GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration - that the container should sleep. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: |- - PreStop is called immediately before a container is terminated due to an - API request or management event such as liveness/startup probe failure, - preemption, resource contention, etc. The handler is not called if the - container crashes or exits. The Pod's termination grace period countdown begins before the - PreStop hook is executed. Regardless of the outcome of the handler, the - container will eventually terminate within the Pod's termination grace - period (unless delayed by finalizers). Other management of the container blocks until the hook completes - or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks - properties: - exec: - description: Exec specifies a command - to execute in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - httpGet: - description: HTTPGet specifies an HTTP - GET request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - sleep: - description: Sleep represents a duration - that the container should sleep. - properties: - seconds: - description: Seconds is the number - of seconds to sleep. - format: int64 - type: integer - required: - - seconds - type: object - tcpSocket: - description: |- - Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept - for backward compatibility. There is no validation of this field and - lifecycle hooks will fail at runtime when it is specified. - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: |- - Periodic probe of container liveness. - Container will be restarted if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute - in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection - to a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - name: - description: |- - Name of the container specified as a DNS_LABEL. - Each container in a pod must have a unique name (DNS_LABEL). - Cannot be updated. - type: string - ports: - description: |- - List of ports to expose from the container. Not specifying a port here - DOES NOT prevent that port from being exposed. Any port which is - listening on the default "0.0.0.0" address inside a container will be - accessible from the network. - Modifying this array with strategic merge patch may corrupt the data. - For more information See https://github.com/kubernetes/kubernetes/issues/108255. - Cannot be updated. - items: - description: ContainerPort represents a network - port in a single container. - properties: - containerPort: - description: |- - Number of port to expose on the pod's IP address. - This must be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the external - port to. - type: string - hostPort: - description: |- - Number of port to expose on the host. - If specified, this must be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this must match ContainerPort. - Most containers do not need this. - format: int32 - type: integer - name: - description: |- - If specified, this must be an IANA_SVC_NAME and unique within the pod. Each - named port in a pod must have a unique name. Name for the port that can be - referred to by services. - type: string - protocol: - default: TCP - description: |- - Protocol for port. Must be UDP, TCP, or SCTP. - Defaults to "TCP". - type: string - required: - - containerPort - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: |- - Periodic probe of container service readiness. - Container will be removed from service endpoints if the probe fails. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute - in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection - to a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - resizePolicy: - description: Resources resize policy for the container. - items: - description: ContainerResizePolicy represents - resource resize policy for the container. - properties: - resourceName: - description: |- - Name of the resource to which this resource resize policy applies. - Supported values: cpu, memory. - type: string - restartPolicy: - description: |- - Restart policy to apply when specified resource is resized. - If not specified, it defaults to NotRequired. - type: string - required: - - resourceName - - restartPolicy - type: object - type: array - x-kubernetes-list-type: atomic - resources: - description: |- - Compute Resources required by this container. - Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - RestartPolicy defines the restart behavior of individual containers in a pod. - This field may only be set for init containers, and the only allowed value is "Always". - For non-init containers or when this field is not specified, - the restart behavior is defined by the Pod's restart policy and the container type. - Setting the RestartPolicy as "Always" for the init container will have the following effect: - this init container will be continually restarted on - exit until all regular containers have terminated. Once all regular - containers have completed, all init containers with restartPolicy "Always" - will be shut down. This lifecycle differs from normal init containers and - is often referred to as a "sidecar" container. Although this init - container still starts in the init container sequence, it does not wait - for the container to complete before proceeding to the next init - container. Instead, the next init container starts immediately after this - init container is started, or after any startupProbe has successfully - completed. - type: string - securityContext: - description: |- - SecurityContext defines the security options the container should be run with. - If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - properties: - allowPrivilegeEscalation: - description: |- - AllowPrivilegeEscalation controls whether a process can gain more - privileges than its parent process. This bool directly controls if - the no_new_privs flag will be set on the container process. - AllowPrivilegeEscalation is true always when the container is: - 1) run as Privileged - 2) has CAP_SYS_ADMIN - Note that this field cannot be set when spec.os.name is windows. - type: boolean - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by this container. If set, this profile - overrides the pod's appArmorProfile. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - capabilities: - description: |- - The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the container runtime. - Note that this field cannot be set when spec.os.name is windows. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - drop: - description: Removed capabilities - items: - description: Capability represent POSIX - capabilities type - type: string - type: array - x-kubernetes-list-type: atomic - type: object - privileged: - description: |- - Run container in privileged mode. - Processes in privileged containers are essentially equivalent to root on the host. - Defaults to false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - procMount: - description: |- - procMount denotes the type of proc mount to use for the containers. - The default value is Default which uses the container runtime defaults for - readonly paths and masked paths. - This requires the ProcMountType feature flag to be enabled. - Note that this field cannot be set when spec.os.name is windows. - type: string - readOnlyRootFilesystem: - description: |- - Whether this container has a read-only root filesystem. - Default is false. - Note that this field cannot be set when spec.os.name is windows. - type: boolean - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxOptions: - description: |- - The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label - that applies to the container. - type: string - role: - description: Role is a SELinux role label - that applies to the container. - type: string - type: - description: Type is a SELinux type label - that applies to the container. - type: string - user: - description: User is a SELinux user label - that applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by this container. If seccomp options are - provided at both the pod & container level, the container options - override the pod options. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options from the PodSecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is - the name of the GMSA credential spec - to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: |- - StartupProbe indicates that the Pod has successfully initialized. - If specified, no other probes are executed until this completes successfully. - If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. - This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, - when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - properties: - exec: - description: Exec specifies a command to execute - in the container. - properties: - command: - description: |- - Command is the command line to execute inside the container, the working directory for the - command is root ('/') in the container's filesystem. The command is simply exec'd, it is - not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use - a shell, you need to explicitly call out to that shell. - Exit status of 0 is treated as live/healthy and non-zero is unhealthy. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - failureThreshold: - description: |- - Minimum consecutive failures for the probe to be considered failed after having succeeded. - Defaults to 3. Minimum value is 1. - format: int32 - type: integer - grpc: - description: GRPC specifies a GRPC HealthCheckRequest. - properties: - port: - description: Port number of the gRPC service. - Number must be in the range 1 to 65535. - format: int32 - type: integer - service: - default: "" - description: |- - Service is the name of the service to place in the gRPC HealthCheckRequest - (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). - - If this is not specified, the default behavior is defined by gRPC. - type: string - required: - - port - type: object - httpGet: - description: HTTPGet specifies an HTTP GET - request to perform. - properties: - host: - description: |- - Host name to connect to, defaults to the pod IP. You probably want to set - "Host" in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to set in - the request. HTTP allows repeated headers. - items: - description: HTTPHeader describes a - custom header to be used in HTTP probes - properties: - name: - description: |- - The header field name. - This will be canonicalized upon output, so case-variant names will be understood as the same header. - type: string - value: - description: The header field value - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - path: - description: Path to access on the HTTP - server. - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Name or number of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: |- - Scheme to use for connecting to the host. - Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: |- - Number of seconds after the container has started before liveness probes are initiated. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - periodSeconds: - description: |- - How often (in seconds) to perform the probe. - Default to 10 seconds. Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: |- - Minimum consecutive successes for the probe to be considered successful after having failed. - Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: TCPSocket specifies a connection - to a TCP port. - properties: - host: - description: 'Optional: Host name to connect - to, defaults to the pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: |- - Number or name of the port to access on the container. - Number must be in the range 1 to 65535. - Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully upon probe failure. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this - value overrides the value provided by the pod spec. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. - Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. - format: int64 - type: integer - timeoutSeconds: - description: |- - Number of seconds after which the probe times out. - Defaults to 1 second. Minimum value is 1. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes - format: int32 - type: integer - type: object - stdin: - description: |- - Whether this container should allocate a buffer for stdin in the container runtime. If this - is not set, reads from stdin in the container will always result in EOF. - Default is false. - type: boolean - stdinOnce: - description: |- - Whether the container runtime should close the stdin channel after it has been opened by - a single attach. When stdin is true the stdin stream will remain open across multiple attach - sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data until the client disconnects, - at which time stdin is closed and remains closed until the container is restarted. If this - flag is false, a container processes that reads from stdin will never receive an EOF. - Default is false - type: boolean - terminationMessagePath: - description: |- - Optional: Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. - Message written is intended to be brief final status, such as an assertion failure message. - Will be truncated by the node if greater than 4096 bytes. The total message length across - all containers will be limited to 12kb. - Defaults to /dev/termination-log. - Cannot be updated. - type: string - terminationMessagePolicy: - description: |- - Indicate how the termination message should be populated. File will use the contents of - terminationMessagePath to populate the container status message on both success and failure. - FallbackToLogsOnError will use the last chunk of container log output if the termination - message file is empty and the container exited with an error. - The log output is limited to 2048 bytes or 80 lines, whichever is smaller. - Defaults to File. - Cannot be updated. - type: string - tty: - description: |- - Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. - Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of block - devices to be used by the container. - items: - description: volumeDevice describes a mapping - of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path inside - of the container that the device will - be mapped to. - type: string - name: - description: name must match the name of - a persistentVolumeClaim in the pod - type: string - required: - - devicePath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - devicePath - x-kubernetes-list-type: map - volumeMounts: - description: |- - Pod volumes to mount into the container's filesystem. - Cannot be updated. - items: - description: VolumeMount describes a mounting - of a Volume within a container. - properties: - mountPath: - description: |- - Path within the container at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: |- - mountPropagation determines how mounts are propagated from the host - to container and the other way around. - When not set, MountPropagationNone is used. - This field is beta in 1.10. - When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified - (which defaults to None). - type: string - name: - description: This must match the Name of - a Volume. - type: string - readOnly: - description: |- - Mounted read-only if true, read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - recursiveReadOnly: - description: |- - RecursiveReadOnly specifies whether read-only mounts should be handled - recursively. - - If ReadOnly is false, this field has no meaning and must be unspecified. - - If ReadOnly is true, and this field is set to Disabled, the mount is not made - recursively read-only. If this field is set to IfPossible, the mount is made - recursively read-only, if it is supported by the container runtime. If this - field is set to Enabled, the mount is made recursively read-only if it is - supported by the container runtime, otherwise the pod will not be started and - an error will be generated to indicate the reason. - - If this field is set to IfPossible or Enabled, MountPropagation must be set to - None (or be unspecified, which defaults to None). - - If this field is not specified, it is treated as an equivalent of Disabled. - type: string - subPath: - description: |- - Path within the volume from which the container's volume should be mounted. - Defaults to "" (volume's root). - type: string - subPathExpr: - description: |- - Expanded path within the volume from which the container's volume should be mounted. - Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. - Defaults to "" (volume's root). - SubPathExpr and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - x-kubernetes-list-map-keys: - - mountPath - x-kubernetes-list-type: map - workingDir: - description: |- - Container's working directory. - If not specified, the container runtime's default will be used, which - might be configured in the container image. - Cannot be updated. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - nodeName: - description: |- - NodeName indicates in which node this pod is scheduled. - If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. - Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. - This field should not be used to express a desire for the pod to be scheduled on a specific node. - https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename - type: string - nodeSelector: - additionalProperties: - type: string - description: |- - NodeSelector is a selector which must be true for the pod to fit on a node. - Selector which must match a node's labels for the pod to be scheduled on that node. - More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - type: object - x-kubernetes-map-type: atomic - os: - description: |- - Specifies the OS of the containers in the pod. - Some pod and container fields are restricted if this is set. - - If the OS field is set to linux, the following fields must be unset: - -securityContext.windowsOptions - - If the OS field is set to windows, following fields must be unset: - - spec.hostPID - - spec.hostIPC - - spec.hostUsers - - spec.securityContext.appArmorProfile - - spec.securityContext.seLinuxOptions - - spec.securityContext.seccompProfile - - spec.securityContext.fsGroup - - spec.securityContext.fsGroupChangePolicy - - spec.securityContext.sysctls - - spec.shareProcessNamespace - - spec.securityContext.runAsUser - - spec.securityContext.runAsGroup - - spec.securityContext.supplementalGroups - - spec.securityContext.supplementalGroupsPolicy - - spec.containers[*].securityContext.appArmorProfile - - spec.containers[*].securityContext.seLinuxOptions - - spec.containers[*].securityContext.seccompProfile - - spec.containers[*].securityContext.capabilities - - spec.containers[*].securityContext.readOnlyRootFilesystem - - spec.containers[*].securityContext.privileged - - spec.containers[*].securityContext.allowPrivilegeEscalation - - spec.containers[*].securityContext.procMount - - spec.containers[*].securityContext.runAsUser - - spec.containers[*].securityContext.runAsGroup - properties: - name: - description: |- - Name is the name of the operating system. The currently supported values are linux and windows. - Additional value may be defined in future and can be one of: - https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration - Clients should expect to handle additional values and treat unrecognized values in this field as os: null - type: string - required: - - name - type: object - overhead: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. - This field will be autopopulated at admission time by the RuntimeClass admission controller. If - the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. - The RuntimeClass admission controller will reject Pod create requests which have the overhead already - set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value - defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. - More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md - type: object - preemptionPolicy: - description: |- - PreemptionPolicy is the Policy for preempting pods with lower priority. - One of Never, PreemptLowerPriority. - Defaults to PreemptLowerPriority if unset. - type: string - priority: - description: |- - The priority value. Various system components use this field to find the - priority of the pod. When Priority Admission Controller is enabled, it - prevents users from setting this field. The admission controller populates - this field from PriorityClassName. - The higher the value, the higher the priority. - format: int32 - type: integer - priorityClassName: - description: |- - If specified, indicates the pod's priority. "system-node-critical" and - "system-cluster-critical" are two special keywords which indicate the - highest priorities with the former being the highest priority. Any other - name must be defined by creating a PriorityClass object with that name. - If not specified, the pod priority will be default or zero if there is no - default. - type: string - readinessGates: - description: |- - If specified, all readiness gates will be evaluated for pod readiness. - A pod is ready when all its containers are ready AND - all conditions specified in the readiness gates have status equal to "True" - More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates - items: - description: PodReadinessGate contains the reference - to a pod condition - properties: - conditionType: - description: ConditionType refers to a condition - in the pod's condition list with matching type. - type: string - required: - - conditionType - type: object - type: array - x-kubernetes-list-type: atomic - resourceClaims: - description: |- - ResourceClaims defines which ResourceClaims must be allocated - and reserved before the Pod is allowed to start. The resources - will be made available to those containers which consume them - by name. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. - items: - description: |- - PodResourceClaim references exactly one ResourceClaim, either directly - or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim - for the pod. - - It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. - Containers that need access to the ResourceClaim reference it with this name. - properties: - name: - description: |- - Name uniquely identifies this resource claim inside the pod. - This must be a DNS_LABEL. - type: string - resourceClaimName: - description: |- - ResourceClaimName is the name of a ResourceClaim object in the same - namespace as this pod. - - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. - type: string - resourceClaimTemplateName: - description: |- - ResourceClaimTemplateName is the name of a ResourceClaimTemplate - object in the same namespace as this pod. - - The template will be used to create a new ResourceClaim, which will - be bound to this pod. When this pod is deleted, the ResourceClaim - will also be deleted. The pod name and resource name, along with a - generated component, will be used to form a unique name for the - ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. - - This field is immutable and no changes will be made to the - corresponding ResourceClaim by the control plane after creating the - ResourceClaim. - - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - resources: - description: |- - Resources is the total amount of CPU and Memory resources required by all - containers in the pod. It supports specifying Requests and Limits for - "cpu" and "memory" resource names only. ResourceClaims are not supported. - - This field enables fine-grained control over resource allocation for the - entire pod, allowing resource sharing among containers in a pod. - - This is an alpha field and requires enabling the PodLevelResources feature - gate. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry - in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - restartPolicy: - description: |- - Restart policy for all containers within the pod. - One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. - Default to Always. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy - type: string - runtimeClassName: - description: |- - RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used - to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. - If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an - empty definition that uses the default runtime handler. - More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class - type: string - schedulerName: - description: |- - If specified, the pod will be dispatched by specified scheduler. - If not specified, the pod will be dispatched by default scheduler. - type: string - schedulingGates: - description: |- - SchedulingGates is an opaque list of values that if specified will block scheduling the pod. - If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the - scheduler will not attempt to schedule the pod. - - SchedulingGates can only be set at pod creation time, and be removed only afterwards. - items: - description: PodSchedulingGate is associated to a - Pod to guard its scheduling. - properties: - name: - description: |- - Name of the scheduling gate. - Each scheduling gate must have a unique name field. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - securityContext: - description: |- - SecurityContext holds pod-level security attributes and common container settings. - Optional: Defaults to empty. See type description for default values of each field. - properties: - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: - description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - fsGroup: - description: |- - A special supplemental group that applies to all containers in a pod. - Some volume types allow the Kubelet to change the ownership of that volume - to be owned by the pod: - - 1. The owning GID will be the FSGroup - 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) - 3. The permission bits are OR'd with rw-rw---- - - If unset, the Kubelet will not modify the ownership and permissions of any volume. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - fsGroupChangePolicy: - description: |- - fsGroupChangePolicy defines behavior of changing ownership and permission of the volume - before being exposed inside Pod. This field will only apply to - volume types which support fsGroup based ownership(and permissions). - It will have no effect on ephemeral volume types such as: secret, configmaps - and emptydir. - Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. - Note that this field cannot be set when spec.os.name is windows. - type: string - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxChangePolicy: - description: |- - seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. - It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. - Valid values are "MountOption" and "Recursive". - - "Recursive" means relabeling of all files on all Pod volumes by the container runtime. - This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. - - "MountOption" mounts all eligible Pod volumes with `-o context` mount option. - This requires all Pods that share the same volume to use the same SELinux label. - It is not possible to share the same volume among privileged and unprivileged Pods. - Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes - whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their - CSIDriver instance. Other volumes are always re-labelled recursively. - "MountOption" value is allowed only when SELinuxMount feature gate is enabled. - - If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. - If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes - and "Recursive" for all other volumes. - - This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. - - All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. - Note that this field cannot be set when spec.os.name is windows. - type: string - seLinuxOptions: - description: |- - The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in SecurityContext. If set in - both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that - applies to the container. - type: string - role: - description: Role is a SELinux role label that - applies to the container. - type: string - type: - description: Type is a SELinux type label that - applies to the container. - type: string - user: - description: User is a SELinux user label that - applies to the container. - type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: - description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: - description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: - - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. - type: string - required: - - type - type: object - supplementalGroups: - description: |- - A list of groups applied to the first process run in each container, in - addition to the container's primary GID and fsGroup (if specified). If - the SupplementalGroupsPolicy feature is enabled, the - supplementalGroupsPolicy field determines whether these are in addition - to or instead of any group memberships defined in the container image. - If unspecified, no additional groups are added, though group memberships - defined in the container image may still be used, depending on the - supplementalGroupsPolicy field. - Note that this field cannot be set when spec.os.name is windows. - items: - format: int64 - type: integer - type: array - x-kubernetes-list-type: atomic - supplementalGroupsPolicy: - description: |- - Defines how supplemental groups of the first container processes are calculated. - Valid values are "Merge" and "Strict". If not specified, "Merge" is used. - (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled - and the container runtime must implement support for this feature. - Note that this field cannot be set when spec.os.name is windows. - type: string - sysctls: - description: |- - Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported - sysctls (by the container runtime) might fail to launch. - Note that this field cannot be set when spec.os.name is windows. - items: - description: Sysctl defines a kernel parameter - to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - x-kubernetes-list-type: atomic - windowsOptions: - description: |- - The Windows specific settings applied to all containers. - If unspecified, the options within a container's SecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name - of the GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - serviceAccount: - description: |- - DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. - Deprecated: Use serviceAccountName instead. - type: string - serviceAccountName: - description: |- - ServiceAccountName is the name of the ServiceAccount to use to run this pod. - More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ - type: string - setHostnameAsFQDN: - description: |- - If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). - In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). - In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. - If a pod does not have FQDN, this has no effect. - Default to false. - type: boolean - shareProcessNamespace: - description: |- - Share a single process namespace between all of the containers in a pod. - When this is set containers will be able to view and signal processes from other containers - in the same pod, and the first process in each container will not be assigned PID 1. - HostPID and ShareProcessNamespace cannot both be set. - Optional: Default to false. - type: boolean - subdomain: - description: |- - If specified, the fully qualified Pod hostname will be "...svc.". - If not specified, the pod will not have a domainname at all. - type: string - terminationGracePeriodSeconds: - description: |- - Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. - Value must be non-negative integer. The value zero indicates stop immediately via - the kill signal (no opportunity to shut down). - If this value is nil, the default grace period will be used instead. - The grace period is the duration in seconds after the processes running in the pod are sent - a termination signal and the time when the processes are forcibly halted with a kill signal. - Set this value longer than the expected cleanup time for your process. - Defaults to 30 seconds. - format: int64 - type: integer - tolerations: - description: If specified, the pod's tolerations. - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - topologySpreadConstraints: - description: |- - TopologySpreadConstraints describes how a group of pods ought to spread across topology - domains. Scheduler will schedule pods in a way which abides by the constraints. - All topologySpreadConstraints are ANDed. - items: - description: TopologySpreadConstraint specifies how - to spread matching pods among the given topology. - properties: - labelSelector: - description: |- - LabelSelector is used to find matching pods. - Pods that match this label selector are counted to determine the number of pods - in their corresponding topology domain. - properties: - matchExpressions: - description: matchExpressions is a list of - label selector requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select the pods over which - spreading will be calculated. The keys are used to lookup values from the - incoming pod labels, those key-value labels are ANDed with labelSelector - to select the group of existing pods over which spreading will be calculated - for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - MatchLabelKeys cannot be set when LabelSelector isn't set. - Keys that don't exist in the incoming pod labels will - be ignored. A null or empty list means only match against labelSelector. - - This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxSkew: - description: |- - MaxSkew describes the degree to which pods may be unevenly distributed. - When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference - between the number of matching pods in the target topology and the global minimum. - The global minimum is the minimum number of matching pods in an eligible domain - or zero if the number of eligible domains is less than MinDomains. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 2/2/1: - In this case, the global minimum is 1. - | zone1 | zone2 | zone3 | - | P P | P P | P | - - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; - scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) - violate MaxSkew(1). - - if MaxSkew is 2, incoming pod can be scheduled onto any zone. - When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence - to topologies that satisfy it. - It's a required field. Default value is 1 and 0 is not allowed. - format: int32 - type: integer - minDomains: - description: |- - MinDomains indicates a minimum number of eligible domains. - When the number of eligible domains with matching topology keys is less than minDomains, - Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. - And when the number of eligible domains with matching topology keys equals or greater than minDomains, - this value has no effect on scheduling. - As a result, when the number of eligible domains is less than minDomains, - scheduler won't schedule more than maxSkew Pods to those domains. - If value is nil, the constraint behaves as if MinDomains is equal to 1. - Valid values are integers greater than 0. - When value is not nil, WhenUnsatisfiable must be DoNotSchedule. - - For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same - labelSelector spread as 2/2/2: - | zone1 | zone2 | zone3 | - | P P | P P | P P | - The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. - In this situation, new pod with the same labelSelector cannot be scheduled, - because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, - it will violate MaxSkew. - format: int32 - type: integer - nodeAffinityPolicy: - description: |- - NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector - when calculating pod topology spread skew. Options are: - - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. - - If this value is nil, the behavior is equivalent to the Honor policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. - type: string - nodeTaintsPolicy: - description: |- - NodeTaintsPolicy indicates how we will treat node taints when calculating - pod topology spread skew. Options are: - - Honor: nodes without taints, along with tainted nodes for which the incoming pod - has a toleration, are included. - - Ignore: node taints are ignored. All nodes are included. - - If this value is nil, the behavior is equivalent to the Ignore policy. - This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. - type: string - topologyKey: - description: |- - TopologyKey is the key of node labels. Nodes that have a label with this key - and identical values are considered to be in the same topology. - We consider each as a "bucket", and try to put balanced number - of pods into each bucket. - We define a domain as a particular instance of a topology. - Also, we define an eligible domain as a domain whose nodes meet the requirements of - nodeAffinityPolicy and nodeTaintsPolicy. - e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. - And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. - It's a required field. - type: string - whenUnsatisfiable: - description: |- - WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy - the spread constraint. - - DoNotSchedule (default) tells the scheduler not to schedule it. - - ScheduleAnyway tells the scheduler to schedule the pod in any location, - but giving higher precedence to topologies that would help reduce the - skew. - A constraint is considered "Unsatisfiable" for an incoming pod - if and only if every possible node assignment for that pod would violate - "MaxSkew" on some topology. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 3/1/1: - | zone1 | zone2 | zone3 | - | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled - to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies - MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler - won't make it *more* imbalanced. - It's a required field. - type: string - required: - - maxSkew - - topologyKey - - whenUnsatisfiable - type: object - type: array - x-kubernetes-list-map-keys: - - topologyKey - - whenUnsatisfiable - x-kubernetes-list-type: map - volumes: - description: |- - List of volumes that can be mounted by containers belonging to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes - items: - description: Volume represents a named volume in a - pod that may be accessed by any container in the - pod. - properties: - awsElasticBlockStore: - description: |- - awsElasticBlockStore represents an AWS Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - Deprecated: AWSElasticBlockStore is deprecated. All operations for the in-tree - awsElasticBlockStore type are redirected to the ebs.csi.aws.com CSI driver. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - format: int32 - type: integer - readOnly: - description: |- - readOnly value true will force the readOnly setting in VolumeMounts. - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: boolean - volumeID: - description: |- - volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). - More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - type: string - required: - - volumeID - type: object - azureDisk: - description: |- - azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. - Deprecated: AzureDisk is deprecated. All operations for the in-tree azureDisk type - are redirected to the disk.csi.azure.com CSI driver. - properties: - cachingMode: - description: 'cachingMode is the Host Caching - mode: None, Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data - disk in the blob storage - type: string - diskURI: - description: diskURI is the URI of data disk - in the blob storage - type: string - fsType: - default: ext4 - description: |- - fsType is Filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - kind: - description: 'kind expected values are Shared: - multiple blob disks per storage account Dedicated: - single blob disk per storage account Managed: - azure managed data disk (only in managed - availability set). defaults to shared' - type: string - readOnly: - default: false - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: |- - azureFile represents an Azure File Service mount on the host and bind mount to the pod. - Deprecated: AzureFile is deprecated. All operations for the in-tree azureFile type - are redirected to the file.csi.azure.com CSI driver. - properties: - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret - that contains Azure Storage Account Name - and Key - type: string - shareName: - description: shareName is the azure share - Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: |- - cephFS represents a Ceph FS mount on the host that shares a pod's lifetime. - Deprecated: CephFS is deprecated and the in-tree cephfs type is no longer supported. - properties: - monitors: - description: |- - monitors is Required: Monitors is a collection of Ceph monitors - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - items: - type: string - type: array - x-kubernetes-list-type: atomic - path: - description: 'path is Optional: Used as the - mounted root, rather than the full Ceph - tree, default is /' - type: string - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: boolean - secretFile: - description: |- - secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - secretRef: - description: |- - secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - user: - description: |- - user is optional: User is the rados user name, default is admin - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it - type: string - required: - - monitors - type: object - cinder: - description: |- - cinder represents a cinder volume attached and mounted on kubelets host machine. - Deprecated: Cinder is deprecated. All operations for the in-tree cinder type - are redirected to the cinder.csi.openstack.org CSI driver. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: boolean - secretRef: - description: |- - secretRef is optional: points to a secret object containing parameters used to connect - to OpenStack. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - volumeID: - description: |- - volumeID used to identify the volume in cinder. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap - that should populate this volume - properties: - defaultMode: - description: |- - defaultMode is optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path - within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional specify whether the - ConfigMap or its keys must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - csi: - description: csi (Container Storage Interface) - represents ephemeral storage that is handled - by certain external CSI drivers. - properties: - driver: - description: |- - driver is the name of the CSI driver that handles this volume. - Consult with your admin for the correct name as registered in the cluster. - type: string - fsType: - description: |- - fsType to mount. Ex. "ext4", "xfs", "ntfs". - If not provided, the empty value is passed to the associated CSI driver - which will determine the default filesystem to apply. - type: string - nodePublishSecretRef: - description: |- - nodePublishSecretRef is a reference to the secret object containing - sensitive information to pass to the CSI driver to complete the CSI - NodePublishVolume and NodeUnpublishVolume calls. - This field is optional, and may be empty if no secret is required. If the - secret object contains more than one secret, all secret references are passed. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - readOnly: - description: |- - readOnly specifies a read-only configuration for the volume. - Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: |- - volumeAttributes stores driver-specific properties that are passed to the CSI - driver. Consult your driver's documentation for supported values. - type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API - about the pod that should populate this volume - properties: - defaultMode: - description: |- - Optional: mode bits to use on created files by default. Must be a - Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: Items is a list of downward API - volume file - items: - description: DownwardAPIVolumeFile represents - information to create the file containing - the pod field - properties: - fieldRef: - description: 'Required: Selects a field - of the pod: only annotations, labels, - name, namespace and uid are supported.' - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path is the - relative path name of the file to - be created. Must not be absolute or - contain the ''..'' path. Must be utf-8 - encoded. The first item of the relative - path must not start with ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - emptyDir: - description: |- - emptyDir represents a temporary directory that shares a pod's lifetime. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - properties: - medium: - description: |- - medium represents what type of storage medium should back this directory. - The default is "" which means to use the node's default medium. - Must be an empty string (default) or Memory. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: |- - sizeLimit is the total amount of local storage required for this EmptyDir volume. - The size limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir would be the minimum value between - the SizeLimit specified here and the sum of memory limits of all containers in a pod. - The default is nil which means that the limit is undefined. - More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: |- - ephemeral represents a volume that is handled by a cluster storage driver. - The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, - and deleted when the pod is removed. - - Use this if: - a) the volume is only needed while the pod runs, - b) features of normal volumes like restoring from snapshot or capacity - tracking are needed, - c) the storage driver is specified through a storage class, and - d) the storage driver supports dynamic volume provisioning through - a PersistentVolumeClaim (see EphemeralVolumeSource for more - information on the connection between this volume type - and PersistentVolumeClaim). - - Use PersistentVolumeClaim or one of the vendor-specific - APIs for volumes that persist for longer than the lifecycle - of an individual pod. - - Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to - be used that way - see the documentation of the driver for - more information. - - A pod can use both types of ephemeral volumes and - persistent volumes at the same time. - properties: - volumeClaimTemplate: - description: |- - Will be used to create a stand-alone PVC to provision the volume. - The pod in which this EphemeralVolumeSource is embedded will be the - owner of the PVC, i.e. the PVC will be deleted together with the - pod. The name of the PVC will be `-` where - `` is the name from the `PodSpec.Volumes` array - entry. Pod validation will reject the pod if the concatenated name - is not valid for a PVC (for example, too long). - - An existing PVC with that name that is not owned by the pod - will *not* be used for the pod to avoid using an unrelated - volume by mistake. Starting the pod is then blocked until - the unrelated PVC is removed. If such a pre-created PVC is - meant to be used by the pod, the PVC has to updated with an - owner reference to the pod once the pod exists. Normally - this should not be necessary, but it may be useful when - manually reconstructing a broken cluster. - - This field is read-only and no changes will be made by Kubernetes - to the PVC after it has been created. - - Required, must not be nil. - properties: - metadata: - description: |- - May contain labels and annotations that will be copied into the PVC - when creating it. No other fields are allowed and will be rejected during - validation. - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: |- - The specification for the PersistentVolumeClaim. The entire content is - copied unchanged into the PVC that gets created from this - template. The same fields as in a PersistentVolumeClaim - are also valid here. - properties: - accessModes: - description: |- - accessModes contains the desired access modes the volume should have. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 - items: - type: string - type: array - x-kubernetes-list-type: atomic - dataSource: - description: |- - dataSource field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) - If the provisioner or an external controller can support the specified data source, - it will create a new volume based on the contents of the specified data source. - When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, - and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. - If the namespace is specified, then dataSourceRef will not be copied to dataSource. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type - of resource being referenced - type: string - name: - description: Name is the name - of resource being referenced - type: string - required: - - kind - - name - type: object - x-kubernetes-map-type: atomic - dataSourceRef: - description: |- - dataSourceRef specifies the object from which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a non-empty API group (non - core object) or a PersistentVolumeClaim object. - When this field is specified, volume binding will only succeed if the type of - the specified object matches some installed volume populator or dynamic - provisioner. - This field will replace the functionality of the dataSource field and as such - if both fields are non-empty, they must have the same value. For backwards - compatibility, when namespace isn't specified in dataSourceRef, - both fields (dataSource and dataSourceRef) will be set to the same - value automatically if one of them is empty and the other is non-empty. - When namespace is specified in dataSourceRef, - dataSource isn't set to the same value and must be empty. - There are three important differences between dataSource and dataSourceRef: - * While dataSource only allows two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim objects. - * While dataSource ignores disallowed values (dropping them), dataSourceRef - preserves all values, and generates an error if a disallowed value is - specified. - * While dataSource only allows local objects, dataSourceRef allows objects - in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. - (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - properties: - apiGroup: - description: |- - APIGroup is the group for the resource being referenced. - If APIGroup is not specified, the specified Kind must be in the core API group. - For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type - of resource being referenced - type: string - name: - description: Name is the name - of resource being referenced - type: string - namespace: - description: |- - Namespace is the namespace of resource being referenced - Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. - (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: |- - resources represents the minimum resources the volume should have. - If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements - that are lower than previous value but must still be higher than capacity recorded in the - status field of the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - selector: - description: selector is a label query - over volumes to consider for binding. - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - storageClassName: - description: |- - storageClassName is the name of the StorageClass required by the claim. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 - type: string - volumeAttributesClassName: - description: |- - volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. - If specified, the CSI driver will create or update the volume with the attributes defined - in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass - will be applied to the claim but it's not allowed to reset this field to empty string once it is set. - If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass - will be set by the persistentvolume controller if it exists. - If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be - set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource - exists. - More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). - type: string - volumeMode: - description: |- - volumeMode defines what type of volume is required by the claim. - Value of Filesystem is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding - reference to the PersistentVolume - backing this claim. - type: string - type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource - that is attached to a kubelet's host machine - and then exposed to the pod. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - lun: - description: 'lun is Optional: FC target lun - number' - format: int32 - type: integer - readOnly: - description: |- - readOnly is Optional: Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target - worldwide names (WWNs)' - items: - type: string - type: array - x-kubernetes-list-type: atomic - wwids: - description: |- - wwids Optional: FC volume world wide identifiers (wwids) - Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - flexVolume: - description: |- - flexVolume represents a generic volume resource that is - provisioned/attached using an exec based plugin. - Deprecated: FlexVolume is deprecated. Consider using a CSIDriver instead. - properties: - driver: - description: driver is the name of the driver - to use for this volume. - type: string - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field - holds extra command options if any.' - type: object - readOnly: - description: |- - readOnly is Optional: defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef is Optional: secretRef is reference to the secret object containing - sensitive information to pass to the plugin scripts. This may be - empty if no secret object is specified. If the secret object - contains more than one secret, all secrets are passed to the plugin - scripts. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - required: - - driver - type: object - flocker: - description: |- - flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running. - Deprecated: Flocker is deprecated and the in-tree flocker type is no longer supported. - properties: - datasetName: - description: |- - datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker - should be considered as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the - dataset. This is unique identifier of a - Flocker dataset - type: string - type: object - gcePersistentDisk: - description: |- - gcePersistentDisk represents a GCE Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - Deprecated: GCEPersistentDisk is deprecated. All operations for the in-tree - gcePersistentDisk type are redirected to the pd.csi.storage.gke.io CSI driver. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - properties: - fsType: - description: |- - fsType is filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: string - partition: - description: |- - partition is the partition in the volume that you want to mount. - If omitted, the default is to mount by volume name. - Examples: For volume /dev/sda1, you specify the partition as "1". - Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - format: int32 - type: integer - pdName: - description: |- - pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - type: boolean - required: - - pdName - type: object - gitRepo: - description: |- - gitRepo represents a git repository at a particular revision. - Deprecated: GitRepo is deprecated. To provision a container with a git repo, mount an - EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir - into the Pod's container. - properties: - directory: - description: |- - directory is the target directory name. - Must not contain or start with '..'. If '.' is supplied, the volume directory will be the - git repository. Otherwise, if specified, the volume will contain the git repository in - the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for - the specified revision. - type: string - required: - - repository - type: object - glusterfs: - description: |- - glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. - Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. - More info: https://examples.k8s.io/volumes/glusterfs/README.md - properties: - endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: string - path: - description: |- - path is the Glusterfs volume path. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: string - readOnly: - description: |- - readOnly here will force the Glusterfs volume to be mounted with read-only permissions. - Defaults to false. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod - type: boolean - required: - - endpoints - - path - type: object - hostPath: - description: |- - hostPath represents a pre-existing file or directory on the host - machine that is directly exposed to the container. This is generally - used for system agents or other privileged things that are allowed - to see the host machine. Most containers will NOT need this. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - properties: - path: - description: |- - path of the directory on the host. - If the path is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - type: - description: |- - type for HostPath Volume - Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - type: string - required: - - path - type: object - image: - description: |- - image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. - The volume is resolved at pod startup depending on which PullPolicy value is provided: - - - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - - The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. - A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. - The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. - The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. - The volume will be mounted read-only (ro) and non-executable files (noexec). - Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). - The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. - properties: - pullPolicy: - description: |- - Policy for pulling OCI objects. Possible values are: - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - type: string - reference: - description: |- - Required: Image or artifact reference to be used. - Behaves in the same way as pod.spec.containers[*].image. - Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - type: object - iscsi: - description: |- - iscsi represents an ISCSI Disk resource that is attached to a - kubelet's host machine and then exposed to the pod. - More info: https://examples.k8s.io/volumes/iscsi/README.md - properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether - support iSCSI Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether - support iSCSI Session CHAP authentication - type: boolean - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - type: string - initiatorName: - description: |- - initiatorName is the custom iSCSI Initiator Name. - If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface - : will be created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified - Name. - type: string - iscsiInterface: - default: default - description: |- - iscsiInterface is the interface Name that uses an iSCSI transport. - Defaults to 'default' (tcp). - type: string - lun: - description: lun represents iSCSI Target Lun - number. - format: int32 - type: integer - portals: - description: |- - portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - items: - type: string - type: array - x-kubernetes-list-type: atomic - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret - for iSCSI target and initiator authentication - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - targetPortal: - description: |- - targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - description: |- - name of the volume. - Must be a DNS_LABEL and unique within the pod. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - nfs: - description: |- - nfs represents an NFS mount on the host that shares a pod's lifetime - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - properties: - path: - description: |- - path that is exported by the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - readOnly: - description: |- - readOnly here will force the NFS export to be mounted with read-only permissions. - Defaults to false. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: boolean - server: - description: |- - server is the hostname or IP address of the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: |- - persistentVolumeClaimVolumeSource represents a reference to a - PersistentVolumeClaim in the same namespace. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - properties: - claimName: - description: |- - claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims - type: string - readOnly: - description: |- - readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: |- - photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine. - Deprecated: PhotonPersistentDisk is deprecated and the in-tree photonPersistentDisk type is no longer supported. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - pdID: - description: pdID is the ID that identifies - Photon Controller persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: |- - portworxVolume represents a portworx volume attached and mounted on kubelets host machine. - Deprecated: PortworxVolume is deprecated. All operations for the in-tree portworxVolume type - are redirected to the pxd.portworx.com CSI driver when the CSIMigrationPortworx feature-gate - is on. - properties: - fsType: - description: |- - fSType represents the filesystem type to mount - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies - a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources - secrets, configmaps, and downward API - properties: - defaultMode: - description: |- - defaultMode are the mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - sources: - description: |- - sources is the list of volume projections. Each entry in this list - handles one source. - items: - description: |- - Projection that may be projected along with other supported volume types. - Exactly one of these fields must be set. - properties: - clusterTrustBundle: - description: |- - ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field - of ClusterTrustBundle objects in an auto-updating file. - - Alpha, gated by the ClusterTrustBundleProjection feature gate. - - ClusterTrustBundle objects can either be selected by name, or by the - combination of signer name and a label selector. - - Kubelet performs aggressive normalization of the PEM contents written - into the pod filesystem. Esoteric PEM features such as inter-block - comments and block headers are stripped. Certificates are deduplicated. - The ordering of certificates within the file is arbitrary, and Kubelet - may change the order over time. - properties: - labelSelector: - description: |- - Select all ClusterTrustBundles that match this label selector. Only has - effect if signerName is set. Mutually-exclusive with name. If unset, - interpreted as "match nothing". If set but empty, interpreted as "match - everything". - properties: - matchExpressions: - description: matchExpressions - is a list of label selector - requirements. The requirements - are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the - label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - name: - description: |- - Select a single ClusterTrustBundle by object name. Mutually-exclusive - with signerName and labelSelector. - type: string - optional: - description: |- - If true, don't block pod startup if the referenced ClusterTrustBundle(s) - aren't available. If using name, then the named ClusterTrustBundle is - allowed not to exist. If using signerName, then the combination of - signerName and labelSelector is allowed to match zero - ClusterTrustBundles. - type: boolean - path: - description: Relative path from - the volume root to write the bundle. - type: string - signerName: - description: |- - Select all ClusterTrustBundles that match this signer name. - Mutually-exclusive with name. The contents of all selected - ClusterTrustBundles will be unified and deduplicated. - type: string - required: - - path - type: object - configMap: - description: configMap information about - the configMap data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - ConfigMap will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the ConfigMap, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key - to a path within a volume. - properties: - key: - description: key is the key - to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional specify whether - the ConfigMap or its keys must - be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - downwardAPI: - description: downwardAPI information - about the downwardAPI data to project - properties: - items: - description: Items is a list of - DownwardAPIVolume file - items: - description: DownwardAPIVolumeFile - represents information to create - the file containing the pod - field - properties: - fieldRef: - description: 'Required: Selects - a field of the pod: only - annotations, labels, name, - namespace and uid are supported.' - properties: - apiVersion: - description: Version of - the schema the FieldPath - is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the - field to select in the - specified API version. - type: string - required: - - fieldPath - type: object - x-kubernetes-map-type: atomic - mode: - description: |- - Optional: mode bits used to set permissions on this file, must be an octal value - between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: 'Required: Path - is the relative path name - of the file to be created. - Must not be absolute or - contain the ''..'' path. - Must be utf-8 encoded. The - first item of the relative - path must not start with - ''..''' - type: string - resourceFieldRef: - description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. - properties: - containerName: - description: 'Container - name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies - the output format of - the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: - resource to select' - type: string - required: - - resource - type: object - x-kubernetes-map-type: atomic - required: - - path - type: object - type: array - x-kubernetes-list-type: atomic - type: object - secret: - description: secret information about - the secret data to project - properties: - items: - description: |- - items if unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key - to a path within a volume. - properties: - key: - description: key is the key - to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - optional: - description: optional field specify - whether the Secret or its key - must be defined - type: boolean - type: object - x-kubernetes-map-type: atomic - serviceAccountToken: - description: serviceAccountToken is - information about the serviceAccountToken - data to project - properties: - audience: - description: |- - audience is the intended audience of the token. A recipient of a token - must identify itself with an identifier specified in the audience of the - token, and otherwise should reject the token. The audience defaults to the - identifier of the apiserver. - type: string - expirationSeconds: - description: |- - expirationSeconds is the requested duration of validity of the service - account token. As the token approaches expiration, the kubelet volume - plugin will proactively rotate the service account token. The kubelet will - start trying to rotate the token if the token is older than 80 percent of - its time to live or if the token is older than 24 hours.Defaults to 1 hour - and must be at least 10 minutes. - format: int64 - type: integer - path: - description: |- - path is the path relative to the mount point of the file to project the - token into. - type: string - required: - - path - type: object - type: object - type: array - x-kubernetes-list-type: atomic - type: object - quobyte: - description: |- - quobyte represents a Quobyte mount on the host that shares a pod's lifetime. - Deprecated: Quobyte is deprecated and the in-tree quobyte type is no longer supported. - properties: - group: - description: |- - group to map volume access to - Default is no group - type: string - readOnly: - description: |- - readOnly here will force the Quobyte volume to be mounted with read-only permissions. - Defaults to false. - type: boolean - registry: - description: |- - registry represents a single or multiple Quobyte Registry services - specified as a string as host:port pair (multiple entries are separated with commas) - which acts as the central registry for volumes - type: string - tenant: - description: |- - tenant owning the given Quobyte volume in the Backend - Used with dynamically provisioned Quobyte volumes, value is set by the plugin - type: string - user: - description: |- - user to map volume access to - Defaults to serivceaccount user - type: string - volume: - description: volume is a string that references - an already created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: |- - rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. - Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. - More info: https://examples.k8s.io/volumes/rbd/README.md - properties: - fsType: - description: |- - fsType is the filesystem type of the volume that you want to mount. - Tip: Ensure that the filesystem type is supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - type: string - image: - description: |- - image is the rados image name. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - keyring: - default: /etc/ceph/keyring - description: |- - keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - monitors: - description: |- - monitors is a collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - items: - type: string - type: array - x-kubernetes-list-type: atomic - pool: - default: rbd - description: |- - pool is the rados pool name. - Default is rbd. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - readOnly: - description: |- - readOnly here will force the ReadOnly setting in VolumeMounts. - Defaults to false. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: boolean - secretRef: - description: |- - secretRef is name of the authentication secret for RBDUser. If provided - overrides keyring. - Default is nil. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - user: - default: admin - description: |- - user is the rados user name. - Default is admin. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it - type: string - required: - - image - - monitors - type: object - scaleIO: - description: |- - scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. - Deprecated: ScaleIO is deprecated and the in-tree scaleIO type is no longer supported. - properties: - fsType: - default: xfs - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". - Default is "xfs". - type: string - gateway: - description: gateway is the host address of - the ScaleIO API Gateway. - type: string - protectionDomain: - description: protectionDomain is the name - of the ScaleIO Protection Domain for the - configured storage. - type: string - readOnly: - description: |- - readOnly Defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef references to the secret for ScaleIO user and other - sensitive information. If this is not provided, Login operation will fail. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - sslEnabled: - description: sslEnabled Flag enable/disable - SSL communication with Gateway, default - false - type: boolean - storageMode: - default: ThinProvisioned - description: |- - storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage - Pool associated with the protection domain. - type: string - system: - description: system is the name of the storage - system as configured in ScaleIO. - type: string - volumeName: - description: |- - volumeName is the name of a volume already created in the ScaleIO system - that is associated with this volume source. - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - description: |- - secret represents a secret that should populate this volume. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - properties: - defaultMode: - description: |- - defaultMode is Optional: mode bits used to set permissions on created files by default. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values - for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - items: - description: |- - items If unspecified, each key-value pair in the Data field of the referenced - Secret will be projected into the volume as a file whose name is the - key and content is the value. If specified, the listed keys will be - projected into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the Secret, - the volume setup will error unless it is marked optional. Paths must be - relative and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path - within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: |- - mode is Optional: mode bits used to set permissions on this file. - Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - If not specified, the volume defaultMode will be used. - This might be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits set. - format: int32 - type: integer - path: - description: |- - path is the relative path of the file to map the key to. - May not be an absolute path. - May not contain the path element '..'. - May not start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - x-kubernetes-list-type: atomic - optional: - description: optional field specify whether - the Secret or its keys must be defined - type: boolean - secretName: - description: |- - secretName is the name of the secret in the pod's namespace to use. - More info: https://kubernetes.io/docs/concepts/storage/volumes#secret - type: string - type: object - storageos: - description: |- - storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. - Deprecated: StorageOS is deprecated and the in-tree storageos type is no longer supported. - properties: - fsType: - description: |- - fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - readOnly: - description: |- - readOnly defaults to false (read/write). ReadOnly here will force - the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: |- - secretRef specifies the secret to use for obtaining the StorageOS API - credentials. If not specified, default values will be attempted. - properties: - name: - default: "" - description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - type: object - x-kubernetes-map-type: atomic - volumeName: - description: |- - volumeName is the human-readable name of the StorageOS volume. Volume - names are only unique within a namespace. - type: string - volumeNamespace: - description: |- - volumeNamespace specifies the scope of the volume within StorageOS. If no - namespace is specified then the Pod's namespace will be used. This allows the - Kubernetes name scoping to be mirrored within StorageOS for tighter integration. - Set VolumeName to any name to override the default behaviour. - Set to "default" if you are not using namespaces within StorageOS. - Namespaces that do not pre-exist within StorageOS will be created. - type: string - type: object - vsphereVolume: - description: |- - vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine. - Deprecated: VsphereVolume is deprecated. All operations for the in-tree vsphereVolume type - are redirected to the csi.vsphere.vmware.com CSI driver. - properties: - fsType: - description: |- - fsType is filesystem type to mount. - Must be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage - Policy Based Management (SPBM) profile ID - associated with the StoragePolicyName. - type: string - storagePolicyName: - description: storagePolicyName is the storage - Policy Based Management (SPBM) profile name. - type: string - volumePath: - description: volumePath is the path that identifies - vSphere volume vmdk - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - containers - type: object - type: object - topologyRequest: - description: topologyRequest defines the topology request for - the PodSet. - properties: - podIndexLabel: - description: |- - PodIndexLabel indicates the name of the label indexing the pods. - For example, in the context of - - kubernetes job this is: kubernetes.io/job-completion-index - - JobSet: kubernetes.io/job-completion-index (inherited from Job) - - Kubeflow: training.kubeflow.org/replica-index - type: string - preferred: - description: |- - preferred indicates the topology level preferred by the PodSet, as - indicated by the `kueue.x-k8s.io/podset-preferred-topology` PodSet - annotation. - type: string - required: - description: |- - required indicates the topology level required by the PodSet, as - indicated by the `kueue.x-k8s.io/podset-required-topology` PodSet - annotation. - type: string - subGroupCount: - description: |- - SubGroupIndexLabel indicates the count of replicated Jobs (groups) within a PodSet. - For example, in the context of JobSet this value is read from jobset.sigs.k8s.io/replicatedjob-replicas. - format: int32 - type: integer - subGroupIndexLabel: - description: |- - SubGroupIndexLabel indicates the name of the label indexing the instances of replicated Jobs (groups) - within a PodSet. For example, in the context of JobSet this is jobset.sigs.k8s.io/job-index. - type: string - unconstrained: - description: |- - unconstrained indicates that Kueue has the freedom to schedule the PodSet within - the entire available capacity, without constraints on the compactness of the placement. - This is indicated by the `kueue.x-k8s.io/podset-unconstrained-topology` PodSet annotation. - type: boolean - type: object - required: - - count - - template - type: object - x-kubernetes-validations: - - message: minCount should be positive and less or equal to count - rule: 'has(self.minCount) ? self.minCount <= self.count : true' - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - priority: - description: |- - Priority determines the order of access to the resources managed by the - ClusterQueue where the workload is queued. - The priority value is populated from PriorityClassName. - The higher the value, the higher the priority. - If priorityClassName is specified, priority must not be null. - format: int32 - type: integer - priorityClassName: - description: |- - If specified, indicates the workload's priority. - "system-node-critical" and "system-cluster-critical" are two special - keywords which indicate the highest priorities with the former being - the highest priority. Any other name must be defined by creating a - PriorityClass object with that name. If not specified, the workload - priority will be default or zero if there is no default. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - priorityClassSource: - default: "" - description: |- - priorityClassSource determines whether the priorityClass field refers to a pod PriorityClass or kueue.x-k8s.io/workloadpriorityclass. - Workload's PriorityClass can accept the name of a pod priorityClass or a workloadPriorityClass. - When using pod PriorityClass, a priorityClassSource field has the scheduling.k8s.io/priorityclass value. - enum: - - kueue.x-k8s.io/workloadpriorityclass - - scheduling.k8s.io/priorityclass - - "" - type: string - queueName: - description: |- - queueName is the name of the LocalQueue the Workload is associated with. - queueName cannot be changed while .status.admission is not null. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - podSets - type: object - x-kubernetes-validations: - - message: priority should not be nil when priorityClassName is set - rule: 'has(self.priorityClassName) ? has(self.priority) : true' - status: - description: WorkloadStatus defines the observed state of Workload - properties: - accumulatedPastExexcutionTimeSeconds: - description: |- - accumulatedPastExexcutionTimeSeconds holds the total time, in seconds, the workload spent - in Admitted state, in the previous `Admit` - `Evict` cycles. - format: int32 - type: integer - admission: - description: |- - admission holds the parameters of the admission of the workload by a - ClusterQueue. admission can be set back to null, but its fields cannot be - changed once set. - properties: - clusterQueue: - description: clusterQueue is the name of the ClusterQueue that - admitted this workload. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - podSetAssignments: - description: PodSetAssignments hold the admission results for - each of the .spec.podSets entries. - items: - properties: - count: - description: |- - count is the number of pods taken into account at admission time. - This field will not change in case of quota reclaim. - Value could be missing for Workloads created before this field was added, - in that case spec.podSets[*].count value will be used. - format: int32 - minimum: 0 - type: integer - delayedTopologyRequest: - description: |- - delayedTopologyRequest indicates the topology assignment is delayed. - Topology assignment might be delayed in case there is ProvisioningRequest - AdmissionCheck used. - Kueue schedules the second pass of scheduling for each workload with at - least one PodSet which has delayedTopologyRequest=true and without - topologyAssignment. - type: string - flavors: - additionalProperties: - description: ResourceFlavorReference is the name of the - ResourceFlavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - description: Flavors are the flavors assigned to the workload - for each resource. - type: object - name: - default: main - description: Name is the name of the podSet. It should match - one of the names in .spec.podSets. - maxLength: 63 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - resourceUsage: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - resourceUsage keeps track of the total resources all the pods in the podset need to run. - - Beside what is provided in podSet's specs, this calculation takes into account - the LimitRange defaults and RuntimeClass overheads at the moment of admission. - This field will not change in case of quota reclaim. - type: object - topologyAssignment: - description: |- - topologyAssignment indicates the topology assignment divided into - topology domains corresponding to the lowest level of the topology. - The assignment specifies the number of Pods to be scheduled per topology - domain and specifies the node selectors for each topology domain, in the - following way: the node selector keys are specified by the levels field - (same for all domains), and the corresponding node selector value is - specified by the domains.values subfield. If the TopologySpec.Levels field contains - "kubernetes.io/hostname" label, topologyAssignment will contain data only for - this label, and omit higher levels in the topology - - Example: - - topologyAssignment: - levels: - - cloud.provider.com/topology-block - - cloud.provider.com/topology-rack - domains: - - values: [block-1, rack-1] - count: 4 - - values: [block-1, rack-2] - count: 2 - - Here: - - 4 Pods are to be scheduled on nodes matching the node selector: - cloud.provider.com/topology-block: block-1 - cloud.provider.com/topology-rack: rack-1 - - 2 Pods are to be scheduled on nodes matching the node selector: - cloud.provider.com/topology-block: block-1 - cloud.provider.com/topology-rack: rack-2 - - Example: - Below there is an equivalent of the above example assuming, Topology - object defines kubernetes.io/hostname as the lowest level in topology. - Hence we omit higher level of topologies, since the hostname label - is sufficient to explicitly identify a proper node. - - topologyAssignment: - levels: - - kubernetes.io/hostname - domains: - - values: [hostname-1] - count: 4 - - values: [hostname-2] - count: 2 - properties: - domains: - description: |- - domains is a list of topology assignments split by topology domains at - the lowest level of the topology. - items: - properties: - count: - description: |- - count indicates the number of Pods to be scheduled in the topology - domain indicated by the values field. - format: int32 - minimum: 1 - type: integer - values: - description: |- - values is an ordered list of node selector values describing a topology - domain. The values correspond to the consecutive topology levels, from - the highest to the lowest. - items: - type: string - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - required: - - count - - values - type: object - type: array - levels: - description: |- - levels is an ordered list of keys denoting the levels of the assigned - topology (i.e. node label keys), from the highest to the lowest level of - the topology. - items: - type: string - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - required: - - domains - - levels - type: object - required: - - name - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - clusterQueue - - podSetAssignments - type: object - admissionChecks: - description: admissionChecks list all the admission checks required - by the workload and the current status - items: - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - name: - description: name identifies the admission check. - maxLength: 316 - type: string - podSetUpdates: - items: - description: |- - PodSetUpdate contains a list of pod set modifications suggested by AdmissionChecks. - The modifications should be additive only - modifications of already existing keys - or having the same key provided by multiple AdmissionChecks is not allowed and will - result in failure during workload admission. - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - name: - description: Name of the PodSet to modify. Should match - to one of the Workload's PodSets. - maxLength: 63 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - nodeSelector: - additionalProperties: - type: string - type: object - tolerations: - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - maxItems: 8 - type: array - x-kubernetes-validations: - - message: operator must be Exists when 'key' is empty, - which means 'match all values and all keys' - rule: 'self.all(x, !has(x.key) ? x.operator == ''Exists'' - : true)' - - message: effect must be 'NoExecute' when 'tolerationSeconds' - is set - rule: 'self.all(x, has(x.tolerationSeconds) ? x.effect - == ''NoExecute'' : true)' - - message: 'supported toleration values: ''Equal''(default), - ''Exists''' - rule: self.all(x, !has(x.operator) || x.operator in - ['Equal', 'Exists']) - - message: a value must be empty when 'operator' is 'Exists' - rule: 'self.all(x, has(x.operator) && x.operator == - ''Exists'' ? !has(x.value) : true)' - - message: 'supported taint effect values: ''NoSchedule'', - ''PreferNoSchedule'', ''NoExecute''' - rule: self.all(x, !has(x.effect) || x.effect in ['NoSchedule', - 'PreferNoSchedule', 'NoExecute']) - required: - - name - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - state: - description: state of the admissionCheck, one of Pending, Ready, - Retry, Rejected - enum: - - Pending - - Ready - - Retry - - Rejected - type: string - required: - - lastTransitionTime - - message - - name - - state - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - conditions: - description: |- - conditions hold the latest available observations of the Workload - current state. - - The type of the condition could be: - - - Admitted: the Workload was admitted through a ClusterQueue. - - Finished: the associated workload finished running (failed or succeeded). - - PodsReady: at least `.spec.podSets[*].count` Pods are ready or have - succeeded. - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - reclaimablePods: - description: |- - reclaimablePods keeps track of the number pods within a podset for which - the resource reservation is no longer needed. - items: - properties: - count: - description: count is the number of pods for which the requested - resources are no longer needed. - format: int32 - minimum: 0 - type: integer - name: - description: name is the PodSet name. - maxLength: 63 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - count - - name - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - requeueState: - description: |- - requeueState holds the re-queue state - when a workload meets Eviction with PodsReadyTimeout reason. - properties: - count: - description: |- - count records the number of times a workload has been re-queued - When a deactivated (`.spec.activate`=`false`) workload is reactivated (`.spec.activate`=`true`), - this count would be reset to null. - format: int32 - minimum: 0 - type: integer - requeueAt: - description: |- - requeueAt records the time when a workload will be re-queued. - When a deactivated (`.spec.activate`=`false`) workload is reactivated (`.spec.activate`=`true`), - this time would be reset to null. - format: date-time - type: string - type: object - resourceRequests: - description: |- - resourceRequests provides a detailed view of the resources that were - requested by a non-admitted workload when it was considered for admission. - If admission is non-null, resourceRequests will be empty because - admission.resourceUsage contains the detailed information. - items: - properties: - name: - default: main - description: name is the name of the podSet. It should match - one of the names in .spec.podSets. - maxLength: 63 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - resources: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - resources is the total resources all the pods in the podset need to run. - - Beside what is provided in podSet's specs, this value also takes into account - the LimitRange defaults and RuntimeClass overheads at the moment of consideration - and the application of resource.excludeResourcePrefixes and resource.transformations. - type: object - required: - - name - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - schedulingStats: - description: schedulingStats tracks scheduling statistics - properties: - evictions: - description: evictions tracks eviction statistics by reason and - underlyingCause. - items: - properties: - count: - description: count tracks the number of evictions for this - reason and detailed reason. - format: int32 - minimum: 0 - type: integer - reason: - description: reason specifies the programmatic identifier - for the eviction cause. - maxLength: 316 - type: string - underlyingCause: - description: |- - underlyingCause specifies a finer-grained explanation that complements the eviction reason. - This may be an empty string. - maxLength: 316 - type: string - required: - - count - - reason - - underlyingCause - type: object - type: array - x-kubernetes-list-map-keys: - - reason - - underlyingCause - x-kubernetes-list-type: map - type: object - type: object - type: object - x-kubernetes-validations: - - message: podSetAssignments must have the same number of podSets as the spec - rule: 'has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'') && has(self.status.admission) - ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) - : true' - - message: field is immutable - rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'')) ? (oldSelf.spec.priorityClassSource - == self.spec.priorityClassSource) : true' - - message: field is immutable - rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassName) - && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName - == self.spec.priorityClassName) : true' - - message: field is immutable - rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'')) && (has(self.status) - && has(self.status.conditions) && self.status.conditions.exists(c, c.type - == ''QuotaReserved'' && c.status == ''True'')) && has(oldSelf.spec.queueName) - && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName - : true' - - message: maximumExecutionTimeSeconds is immutable while admitted - rule: ((has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == 'Admitted' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) - && self.status.conditions.exists(c, c.type == 'Admitted' && c.status == - 'True')))?((has(oldSelf.spec.maximumExecutionTimeSeconds)?oldSelf.spec.maximumExecutionTimeSeconds:0) - == (has(self.spec.maximumExecutionTimeSeconds)?self.spec.maximumExecutionTimeSeconds:0)):true - served: true - storage: true - subresources: - status: {} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-controller-manager - namespace: kueue-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-leader-election-role - namespace: kueue-system -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch ---- -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.kueue.x-k8s.io/batch-admin: "true" -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-batch-admin-role ---- -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.kueue.x-k8s.io/batch-user: "true" -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-batch-user-role ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-clusterqueue-editor-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - clusterqueues - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - clusterqueues/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-clusterqueue-viewer-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - clusterqueues - verbs: - - get - - list - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - clusterqueues/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-cohort-editor-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - cohorts - verbs: - - create - - delete - - get - - list - - patch - - update - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-cohort-viewer-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - cohorts - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-jaxjob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - jaxjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - jaxjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-jaxjob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - jaxjobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - jaxjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-job-editor-role -rules: -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-job-viewer-role -rules: -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch -- apiGroups: - - batch - resources: - - jobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-jobset-editor-role -rules: -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-jobset-viewer-role -rules: -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets - verbs: - - get - - list - - watch -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-localqueue-editor-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - localqueues - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - localqueues/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-localqueue-viewer-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - localqueues - verbs: - - get - - list - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - localqueues/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-manager-role -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - watch -- apiGroups: - - "" - resources: - - limitranges - - namespaces - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods/finalizers - verbs: - - get - - update -- apiGroups: - - "" - resources: - - pods/status - verbs: - - get - - patch -- apiGroups: - - "" - resources: - - podtemplates - verbs: - - create - - delete - - get - - list - - update - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - update - - watch -- apiGroups: - - admissionregistration.k8s.io - resources: - - mutatingwebhookconfigurations - - validatingwebhookconfigurations - verbs: - - get - - list - - update - - watch -- apiGroups: - - apps - resources: - - deployments - - replicasets - - statefulsets - verbs: - - get - - list - - watch -- apiGroups: - - autoscaling.x-k8s.io - resources: - - provisioningrequests - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - autoscaling.x-k8s.io - resources: - - provisioningrequests/status - verbs: - - get -- apiGroups: - - batch - resources: - - jobs - verbs: - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs/finalizers - - jobs/status - verbs: - - get - - patch - - update -- apiGroups: - - flowcontrol.apiserver.k8s.io - resources: - - flowschemas - - prioritylevelconfigurations - verbs: - - list - - watch -- apiGroups: - - flowcontrol.apiserver.k8s.io - resources: - - flowschemas/status - verbs: - - patch -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets - verbs: - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/finalizers - verbs: - - get - - update -- apiGroups: - - jobset.x-k8s.io - resources: - - jobsets/status - verbs: - - get - - patch - - update -- apiGroups: - - kubeflow.org - resources: - - jaxjobs - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - jaxjobs/finalizers - - mpijobs/finalizers - - paddlejobs/finalizers - - pytorchjobs/finalizers - - tfjobs/finalizers - - xgboostjobs/finalizers - verbs: - - get - - update -- apiGroups: - - kubeflow.org - resources: - - jaxjobs/status - - mpijobs/status - - paddlejobs/status - - pytorchjobs/status - - tfjobs/status - - xgboostjobs/status - verbs: - - get - - patch - - update -- apiGroups: - - kubeflow.org - resources: - - mpijobs - - paddlejobs - - pytorchjobs - - tfjobs - - xgboostjobs - verbs: - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - admissionchecks - - clusterqueues - - cohorts - - localqueues - - workloads - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - admissionchecks/finalizers - - clusterqueues/finalizers - - localqueues/finalizers - - resourceflavors/finalizers - - topologies/finalizers - - workloads/finalizers - verbs: - - update -- apiGroups: - - kueue.x-k8s.io - resources: - - admissionchecks/status - - clusterqueues/status - - cohorts/status - - localqueues/status - - multikueueclusters/status - - workloads/status - verbs: - - get - - patch - - update -- apiGroups: - - kueue.x-k8s.io - resources: - - multikueueclusters - - multikueueconfigs - - provisioningrequestconfigs - - workloadpriorityclasses - verbs: - - get - - list - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - resourceflavors - verbs: - - delete - - get - - list - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - topologies - verbs: - - get - - list - - update - - watch -- apiGroups: - - leaderworkerset.x-k8s.io - resources: - - leaderworkersets - verbs: - - get - - list - - watch -- apiGroups: - - node.k8s.io - resources: - - runtimeclasses - verbs: - - get - - list - - watch -- apiGroups: - - ray.io - resources: - - rayclusters - - rayjobs - verbs: - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - ray.io - resources: - - rayclusters/finalizers - - rayjobs/finalizers - verbs: - - get - - update -- apiGroups: - - ray.io - resources: - - rayclusters/status - - rayjobs/status - verbs: - - get - - patch - - update -- apiGroups: - - scheduling.k8s.io - resources: - - priorityclasses - verbs: - - get - - list - - watch -- apiGroups: - - workload.codeflare.dev - resources: - - appwrappers - verbs: - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - workload.codeflare.dev - resources: - - appwrappers/finalizers - verbs: - - get - - update -- apiGroups: - - workload.codeflare.dev - resources: - - appwrappers/status - verbs: - - get - - patch - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-metrics-auth-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-metrics-reader -rules: -- nonResourceURLs: - - /metrics - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-mpijob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - mpijobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - mpijobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-mpijob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - mpijobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - mpijobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-paddlejob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - paddlejobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - paddlejobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-paddlejob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - paddlejobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - paddlejobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-pending-workloads-cq-viewer-role -rules: -- apiGroups: - - visibility.kueue.x-k8s.io - resources: - - clusterqueues/pendingworkloads - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-pending-workloads-lq-viewer-role -rules: -- apiGroups: - - visibility.kueue.x-k8s.io - resources: - - localqueues/pendingworkloads - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-pytorchjob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-pytorchjob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-raycluster-editor-role -rules: -- apiGroups: - - ray.io - resources: - - rayclusters - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - ray.io - resources: - - rayclusters/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-raycluster-viewer-role -rules: -- apiGroups: - - ray.io - resources: - - rayclusters - verbs: - - get - - list - - watch -- apiGroups: - - ray.io - resources: - - rayclusters/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-rayjob-editor-role -rules: -- apiGroups: - - ray.io - resources: - - rayjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - ray.io - resources: - - rayjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-rayjob-viewer-role -rules: -- apiGroups: - - ray.io - resources: - - rayjobs - verbs: - - get - - list - - watch -- apiGroups: - - ray.io - resources: - - rayjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-resourceflavor-editor-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - resourceflavors - verbs: - - create - - delete - - get - - list - - patch - - update - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-resourceflavor-viewer-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - resourceflavors - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-tfjob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - tfjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - tfjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-tfjob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - tfjobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - tfjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-topology-editor-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - topologies - verbs: - - create - - delete - - get - - list - - patch - - update - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-topology-viewer-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - topologies - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - name: kueue-workload-editor-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - workloads - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - workloads/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-workload-viewer-role -rules: -- apiGroups: - - kueue.x-k8s.io - resources: - - workloads - verbs: - - get - - list - - watch -- apiGroups: - - kueue.x-k8s.io - resources: - - workloads/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-xgboostjob-editor-role -rules: -- apiGroups: - - kubeflow.org - resources: - - xgboostjobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - kubeflow.org - resources: - - xgboostjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - rbac.kueue.x-k8s.io/batch-admin: "true" - rbac.kueue.x-k8s.io/batch-user: "true" - name: kueue-xgboostjob-viewer-role -rules: -- apiGroups: - - kubeflow.org - resources: - - xgboostjobs - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - xgboostjobs/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-visibility-server-auth-reader - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader -subjects: -- kind: ServiceAccount - name: kueue-controller-manager - namespace: kueue-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-leader-election-rolebinding - namespace: kueue-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: kueue-leader-election-role -subjects: -- kind: ServiceAccount - name: kueue-controller-manager - namespace: kueue-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kueue-manager-role -subjects: -- kind: ServiceAccount - name: kueue-controller-manager - namespace: kueue-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-metrics-auth-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kueue-metrics-auth-role -subjects: -- kind: ServiceAccount - name: kueue-controller-manager - namespace: kueue-system ---- -apiVersion: v1 -data: - controller_manager_config.yaml: | - apiVersion: config.kueue.x-k8s.io/v1beta1 - kind: Configuration - health: - healthProbeBindAddress: :8081 - metrics: - bindAddress: :8443 - # enableClusterQueueResources: true - webhook: - port: 9443 - leaderElection: - leaderElect: true - resourceName: c1f6bfd2.kueue.x-k8s.io - controller: - groupKindConcurrency: - Job.batch: 5 - Pod: 5 - Workload.kueue.x-k8s.io: 5 - LocalQueue.kueue.x-k8s.io: 1 - Cohort.kueue.x-k8s.io: 1 - ClusterQueue.kueue.x-k8s.io: 1 - ResourceFlavor.kueue.x-k8s.io: 1 - clientConnection: - qps: 50 - burst: 100 - #pprofBindAddress: :8083 - #waitForPodsReady: - # enable: false - # timeout: 5m - # recoveryTimeout: 3m - # blockAdmission: false - # requeuingStrategy: - # timestamp: Eviction - # backoffLimitCount: null # null indicates infinite requeuing - # backoffBaseSeconds: 60 - # backoffMaxSeconds: 3600 - #manageJobsWithoutQueueName: true - #managedJobsNamespaceSelector: - # matchExpressions: - # - key: kubernetes.io/metadata.name - # operator: NotIn - # values: [ kube-system, kueue-system ] - #internalCertManagement: - # enable: false - # webhookServiceName: "" - # webhookSecretName: "" - integrations: - frameworks: - - "batch/job" - - "kubeflow.org/mpijob" - - "ray.io/rayjob" - - "ray.io/raycluster" - - "jobset.x-k8s.io/jobset" - - "kubeflow.org/paddlejob" - - "kubeflow.org/pytorchjob" - - "kubeflow.org/tfjob" - - "kubeflow.org/xgboostjob" - - "kubeflow.org/jaxjob" - - "workload.codeflare.dev/appwrapper" - # - "pod" - # - "deployment" # requires enabling pod integration - # - "statefulset" # requires enabling pod integration - # - "leaderworkerset.x-k8s.io/leaderworkerset" # requires enabling pod integration - # externalFrameworks: - # - "Foo.v1.example.com" - #fairSharing: - # enable: true - # preemptionStrategies: [LessThanOrEqualToFinalShare, LessThanInitialShare] - #admissionFairSharing: - # usageHalfLifeTime: "168h" # 7 days - # usageSamplingInterval: "5m" - # resourceWeights: # optional, defaults to 1 for all resources if not specified - # cpu: 0 # if you want to completely ignore cpu usage - # memory: 0 # ignore completely memory usage - # example.com/gpu: 100 # and you care only about GPUs usage - #resources: - # excludeResourcePrefixes: [] - # transformations: - # - input: nvidia.com/mig-4g.5gb - # strategy: Replace | Retain - # outputs: - # example.com/accelerator-memory: 5Gi - # example.com/accelerator-gpc: 4 - #objectRetentionPolicies: - # workloads: - # afterFinished: null # null indicates infinite retention, 0s means no retention at all - # afterDeactivatedByKueue: null # null indicates infinite retention, 0s means no retention at all -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-manager-config - namespace: kueue-system ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-webhook-server-cert - namespace: kueue-system ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-controller-manager-metrics-service - namespace: kueue-system -spec: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: 8443 - selector: - control-plane: controller-manager ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-visibility-server - namespace: kueue-system -spec: - ports: - - name: https - port: 443 - protocol: TCP - targetPort: 8082 - selector: - control-plane: controller-manager ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-webhook-service - namespace: kueue-system -spec: - ports: - - port: 443 - protocol: TCP - targetPort: 9443 - selector: - control-plane: controller-manager ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-controller-manager - namespace: kueue-system -spec: - replicas: 1 - selector: - matchLabels: - control-plane: controller-manager - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: manager - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - spec: - containers: - - args: - - --config=/controller_manager_config.yaml - - --zap-log-level=2 - - --feature-gates=TopologyAwareScheduling=true - command: - - /manager - image: registry.k8s.io/kueue/kueue:v0.12.2 - imagePullPolicy: Always - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - ports: - - containerPort: 8443 - name: metrics - protocol: TCP - - containerPort: 8082 - name: visibility - protocol: TCP - - containerPort: 9443 - name: webhook-server - protocol: TCP - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - cpu: "2" - memory: 512Mi - requests: - cpu: 500m - memory: 512Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /visibility - name: visibility - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true - - mountPath: /controller_manager_config.yaml - name: manager-config - subPath: controller_manager_config.yaml - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - serviceAccountName: kueue-controller-manager - terminationGracePeriodSeconds: 10 - volumes: - - emptyDir: {} - name: visibility - - name: cert - secret: - defaultMode: 420 - secretName: kueue-webhook-server-cert - - configMap: - name: kueue-manager-config - name: manager-config - tolerations: - - effect: NoSchedule - key: components.gke.io/gke-managed-components - operator: Equal - value: "true" ---- -apiVersion: apiregistration.k8s.io/v1 -kind: APIService -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: v1beta1.visibility.kueue.x-k8s.io -spec: - group: visibility.kueue.x-k8s.io - groupPriorityMinimum: 100 - insecureSkipTLSVerify: true - service: - name: kueue-visibility-server - namespace: kueue-system - version: v1beta1 - versionPriority: 100 ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-mutating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate--v1-pod - failurePolicy: Fail - name: mpod.kb.io - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kueue-system - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-apps-v1-deployment - failurePolicy: Fail - name: mdeployment.kb.io - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kueue-system - rules: - - apiGroups: - - apps - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - deployments - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-apps-v1-statefulset - failurePolicy: Fail - name: mstatefulset.kb.io - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kueue-system - rules: - - apiGroups: - - apps - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - statefulsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-workload-codeflare-dev-v1beta2-appwrapper - failurePolicy: Fail - name: mappwrapper.kb.io - rules: - - apiGroups: - - workload.codeflare.dev - apiVersions: - - v1beta2 - operations: - - CREATE - resources: - - appwrappers - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-batch-v1-job - failurePolicy: Fail - name: mjob.kb.io - rules: - - apiGroups: - - batch - apiVersions: - - v1 - operations: - - CREATE - resources: - - jobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-jobset-x-k8s-io-v1alpha2-jobset - failurePolicy: Fail - name: mjobset.kb.io - rules: - - apiGroups: - - jobset.x-k8s.io - apiVersions: - - v1alpha2 - operations: - - CREATE - resources: - - jobsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v1-jaxjob - failurePolicy: Fail - name: mjaxjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - resources: - - jaxjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v1-paddlejob - failurePolicy: Fail - name: mpaddlejob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - resources: - - paddlejobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v1-pytorchjob - failurePolicy: Fail - name: mpytorchjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - resources: - - pytorchjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v1-tfjob - failurePolicy: Fail - name: mtfjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - resources: - - tfjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v1-xgboostjob - failurePolicy: Fail - name: mxgboostjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - resources: - - xgboostjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-leaderworkerset-x-k8s-io-v1-leaderworkerset - failurePolicy: Fail - name: mleaderworkerset.kb.io - rules: - - apiGroups: - - leaderworkerset.x-k8s.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - leaderworkersets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kubeflow-org-v2beta1-mpijob - failurePolicy: Fail - name: mmpijob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v2beta1 - operations: - - CREATE - resources: - - mpijobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-ray-io-v1-raycluster - failurePolicy: Fail - name: mraycluster.kb.io - rules: - - apiGroups: - - ray.io - apiVersions: - - v1 - operations: - - CREATE - resources: - - rayclusters - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-ray-io-v1-rayjob - failurePolicy: Fail - name: mrayjob.kb.io - rules: - - apiGroups: - - ray.io - apiVersions: - - v1 - operations: - - CREATE - resources: - - rayjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kueue-x-k8s-io-v1beta1-clusterqueue - failurePolicy: Fail - name: mclusterqueue.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - resources: - - clusterqueues - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kueue-x-k8s-io-v1beta1-resourceflavor - failurePolicy: Fail - name: mresourceflavor.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - resources: - - resourceflavors - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /mutate-kueue-x-k8s-io-v1beta1-workload - failurePolicy: Fail - name: mworkload.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - resources: - - workloads - sideEffects: None ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/name: kueue - control-plane: controller-manager - name: kueue-validating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate--v1-pod - failurePolicy: Fail - name: vpod.kb.io - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kueue-system - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - pods - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-apps-v1-deployment - failurePolicy: Fail - name: vdeployment.kb.io - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kueue-system - rules: - - apiGroups: - - apps - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - deployments - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-apps-v1-statefulset - failurePolicy: Fail - name: vstatefulset.kb.io - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: - - kube-system - - kueue-system - rules: - - apiGroups: - - apps - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - statefulsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-workload-codeflare-dev-v1beta2-appwrapper - failurePolicy: Fail - name: vappwrapper.kb.io - rules: - - apiGroups: - - workload.codeflare.dev - apiVersions: - - v1beta2 - operations: - - CREATE - - UPDATE - resources: - - appwrappers - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-batch-v1-job - failurePolicy: Fail - name: vjob.kb.io - rules: - - apiGroups: - - batch - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - jobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-jobset-x-k8s-io-v1alpha2-jobset - failurePolicy: Fail - name: vjobset.kb.io - rules: - - apiGroups: - - jobset.x-k8s.io - apiVersions: - - v1alpha2 - operations: - - CREATE - - UPDATE - resources: - - jobsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v1-jaxjob - failurePolicy: Fail - name: vjaxjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - jaxjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v1-paddlejob - failurePolicy: Fail - name: vpaddlejob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - paddlejobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v1-pytorchjob - failurePolicy: Fail - name: vpytorchjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - pytorchjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v1-tfjob - failurePolicy: Fail - name: vtfjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - tfjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v1-xgboostjob - failurePolicy: Fail - name: vxgboostjob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - xgboostjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-leaderworkerset-x-k8s-io-v1-leaderworkerset - failurePolicy: Fail - name: vleaderworkerset.kb.io - rules: - - apiGroups: - - leaderworkerset.x-k8s.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - leaderworkersets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kubeflow-org-v2beta1-mpijob - failurePolicy: Fail - name: vmpijob.kb.io - rules: - - apiGroups: - - kubeflow.org - apiVersions: - - v2beta1 - operations: - - CREATE - - UPDATE - resources: - - mpijobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-ray-io-v1-raycluster - failurePolicy: Fail - name: vraycluster.kb.io - rules: - - apiGroups: - - ray.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - rayclusters - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-ray-io-v1-rayjob - failurePolicy: Fail - name: vrayjob.kb.io - rules: - - apiGroups: - - ray.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - rayjobs - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kueue-x-k8s-io-v1beta1-clusterqueue - failurePolicy: Fail - name: vclusterqueue.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - clusterqueues - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kueue-x-k8s-io-v1alpha1-cohort - failurePolicy: Fail - name: vcohort.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1alpha1 - operations: - - CREATE - - UPDATE - resources: - - cohorts - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kueue-x-k8s-io-v1beta1-resourceflavor - failurePolicy: Fail - name: vresourceflavor.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - resourceflavors - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: kueue-webhook-service - namespace: kueue-system - path: /validate-kueue-x-k8s-io-v1beta1-workload - failurePolicy: Fail - name: vworkload.kb.io - rules: - - apiGroups: - - kueue.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - workloads - - workloads/status - sideEffects: None diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/metadata.yaml deleted file mode 100644 index 17bedb471b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/providers.tf b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/providers.tf deleted file mode 100644 index ba9a4af370..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/providers.tf +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -provider "kubectl" { - host = "https://${data.google_container_cluster.gke_cluster.endpoint}" - token = data.google_client_config.default.access_token - cluster_ca_certificate = base64decode(data.google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate) - load_config_file = false - apply_retry_count = 15 # Terraform may apply resources in parallel, leading to potential dependency issues. This retry mechanism ensures that if a resource's dependencies aren't ready, Terraform will attempt to apply it again. -} - -provider "helm" { - kubernetes { - host = "https://${data.google_container_cluster.gke_cluster.endpoint}" - token = data.google_client_config.default.access_token - cluster_ca_certificate = base64decode( - data.google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate, - ) - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/variables.tf deleted file mode 100644 index 61bed61d4f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/variables.tf +++ /dev/null @@ -1,166 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - kueue_supported_versions = ["v0.12.2", "v0.11.4", "v0.10.1", "v0.10.0"] - jobset_supported_versions = ["v0.8.1", "v0.7.2", "v0.5.2"] - gib_supported_versions = ["v1.0.2", "v1.0.3", "v1.0.5", "v1.0.6"] -} - -resource "terraform_data" "kueue_validations" { - lifecycle { - precondition { - condition = !var.kueue.install || contains(local.kueue_supported_versions, var.kueue.version) - error_message = "Supported version of Kueue are ${join(", ", local.kueue_supported_versions)}" - } - } -} - -resource "terraform_data" "jobset_validations" { - lifecycle { - precondition { - condition = !var.jobset.install || contains(local.jobset_supported_versions, var.jobset.version) - error_message = "Supported version of Jobset are ${join(", ", local.jobset_supported_versions)}" - } - } -} - -resource "terraform_data" "gib_validations" { - lifecycle { - precondition { - condition = !var.gib.install || contains(local.gib_supported_versions, var.gib.template_vars.version) - error_message = "Supported version of the NCCL gIB plugin are ${join(", ", local.gib_supported_versions)}" - } - } -} - -resource "terraform_data" "initial_gib_version" { - input = var.gib.install ? var.gib.template_vars.version : null - - lifecycle { - ignore_changes = [input] - } -} - -check "gib_version_changes" { - assert { - # Skip version checking if gIB was not initially or is not currently installed - condition = terraform_data.initial_gib_version.output == null || !var.gib.install || terraform_data.initial_gib_version.output == var.gib.template_vars.version - error_message = "When changing the gIB NCCL plugin version, confirm full rollout and environment consistency. Replace any NCCL env hard coding/caches with set_nccl_env.sh sourcing." - } -} - -variable "project_id" { - description = "The project ID that hosts the gke cluster." - type = string -} - -variable "cluster_id" { - description = "An identifier for the gke cluster resource with format projects//locations//clusters/." - type = string - nullable = false -} - -variable "apply_manifests" { - description = "A list of manifests to apply to GKE cluster using kubectl. For more details see [kubectl module's inputs](kubectl/README.md)." - type = list(object({ - enable = optional(bool, true) - content = optional(string, null) - source = optional(string, null) - template_vars = optional(map(any), null) - server_side_apply = optional(bool, false) - wait_for_rollout = optional(bool, true) - })) - default = [] -} - - -variable "kueue" { - description = "Install and configure [Kueue](https://kueue.sigs.k8s.io/docs/overview/) workload scheduler. A configuration yaml/template file can be provided with config_path to be applied right after kueue installation. If a template file provided, its variables can be set to config_template_vars." - type = object({ - install = optional(bool, false) - version = optional(string, "v0.12.2") - config_path = optional(string, null) - config_template_vars = optional(map(any), null) - }) - default = {} -} - -variable "gke_cluster_exists" { - description = "A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations." - type = bool - default = false -} - -variable "jobset" { - description = "Install [Jobset](https://github.com/kubernetes-sigs/jobset) which manages a group of K8s [jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/) as a unit." - type = object({ - install = optional(bool, false) - version = optional(string, "v0.7.2") - }) - default = {} -} - - -variable "gpu_operator" { - description = "Install [GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) which uses the [Kubernetes operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) to automate the management of all NVIDIA software components needed to provision GPU." - type = object({ - install = optional(bool, false) - version = optional(string, "v25.3.0") - }) - default = {} -} - -variable "nvidia_dra_driver" { - description = "Installs [Nvidia DRA driver](https://github.com/NVIDIA/k8s-dra-driver-gpu) which supports Dynamic Resource Allocation for NVIDIA GPUs in Kubernetes" - type = object({ - install = optional(bool, false) - version = optional(string, "v25.3.0-rc.3") - }) - default = {} -} - -variable "gib" { - description = "Install the NCCL gIB plugin" - type = object({ - install = bool - path = string - template_vars = object({ - image = optional(string, "us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib") - version = string - node_affinity = optional(any, { - requiredDuringSchedulingIgnoredDuringExecution = { - nodeSelectorTerms = [{ - matchExpressions = [{ - key = "cloud.google.com/gke-gpu", - operator = "In", - values = ["true"] - }] - }] - } - }) - accelerator_count = number - }) - }) - default = { - install = false - path = "" - template_vars = { - version = "" - accelerator_count = 0 - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/versions.tf deleted file mode 100644 index e5cc65acb1..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/management/kubectl-apply/versions.tf +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = "> 5.0" - } - kubectl = { - source = "gavinbunney/kubectl" - version = ">= 1.7.0" - } - helm = { - source = "hashicorp/helm" - version = "~> 2.17" - } - } - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:kubectl-apply/v1.37.2" - } - - required_version = ">= 1.3" -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/README.md b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/README.md deleted file mode 100644 index 949346525e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/README.md +++ /dev/null @@ -1,86 +0,0 @@ -## Description - -Creates a [monitoring dashboard][gcp-dash] for the HPC cluster deployment. The -module includes a default HPC-focused dashboard with the ability to add custom -widgets as well as the option to add an empty dashboard and add widgets as -needed. - -[gcp-dash]: https://cloud.google.com/monitoring/charts/predefined-dashboards - -## Example - -```yaml -- id: hpc_dash - source: modules/monitoring/dashboard - settings: - widgets: - - | - { - "text": { - "content": "## Header", - "format": "MARKDOWN" - }, - "title": "Custom Text Block Widget" - } -``` - -This module creates a dashboard based on the HPC dashboard (default) with an -extra text widget added as a multi-line string representing a JSON block. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_monitoring_dashboard.dashboard](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/monitoring_dashboard) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [base\_dashboard](#input\_base\_dashboard) | Baseline dashboard template, select from HPC or Empty | `string` | `"HPC"` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the monitoring dashboard instance. Key-value pairs. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [title](#input\_title) | Title of the created dashboard | `string` | `"Cluster Toolkit Dashboard"` | no | -| [widgets](#input\_widgets) | List of additional widgets to add to the base dashboard. | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [instructions](#output\_instructions) | Instructions for accessing the monitoring dashboard | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl deleted file mode 100644 index f25cbbd2c6..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/dashboards/Empty.json.tpl +++ /dev/null @@ -1,17 +0,0 @@ -{ - "displayName": "${title}: ${deployment_name}", - "gridLayout": { - "columns": 2, - "widgets": [ - { - "text": { - "content": "Metrics from the ${deployment_name} deployment of the Cluster Toolkit.", - "format": "MARKDOWN" - }, - "title": "${title}" - }%{ for widget in widgets ~}, - ${widget} - %{endfor ~} - ] - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl deleted file mode 100644 index 5b20435a9a..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/dashboards/HPC.json.tpl +++ /dev/null @@ -1,595 +0,0 @@ -{ - "displayName": "${title}: ${deployment_name}", - "labels": ${jsonencode(labels)}, - "gridLayout": { - "columns": 2, - "widgets": [ - { - "text": { - "content": "HPC metrics from the ${deployment_name} deployment of the Cluster Toolkit.", - "format": "MARKDOWN" - }, - "title": "${title}" - }, - { - "title": "VM Instance - Memory utilization", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"agent.googleapis.com/memory/percent_used\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - CPU Utilization", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"", - "pickTimeSeriesFilter": { - "direction": "TOP", - "numTimeSeries": 20, - "rankingMethod": "METHOD_MEAN" - } - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - CPU utilization (agent)", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"agent.googleapis.com/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - }, - "unitOverride": "%" - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Disk read operations", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/disk/read_ops_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Disk write operations", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/disk/write_ops_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Disk Read Bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"agent.googleapis.com/disk/read_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Disk Write Bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"agent.googleapis.com/disk/write_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "Throttled read bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/disk/throttled_read_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "Throttled write bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/disk/throttled_write_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Received packets", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/network/received_packets_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "VM Instance - Sent packets", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/network/sent_packets_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "VM Instance - Received bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/network/received_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Sent bytes", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_MEAN", - "groupByFields": [ - "metric.label.\"instance_name\"", - "metric.label.\"loadbalanced\"", - "resource.label.\"project_id\"", - "resource.label.\"instance_id\"", - "resource.label.\"zone\"" - ], - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/network/sent_bytes_count\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"", - "secondaryAggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - } - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "GCE VM Instance - Network Traffic Bytes (agent)", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"agent.googleapis.com/interface/traffic\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "Network Packets (agent)", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_RATE" - }, - "filter": "metric.type=\"agent.googleapis.com/interface/packets\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "TCP connections", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"agent.googleapis.com/network/tcp_connections\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - }, - "unitOverride": "1" - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "VM Instance - CPU utilization for steal", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "STACKED_BAR", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MAX" - }, - "filter": "metric.type=\"agent.googleapis.com/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\" metric.label.\"cpu_state\"=\"steal\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - { - "title": "VM Instance - CPU utilization [MEAN]", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" resource.type=\"gce_instance\" metadata.user_labels.\"ghpc_deployment\"=\"${deployment_name}\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }%{ for widget in widgets ~}, - ${widget} - %{endfor ~} - ] - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/main.tf b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/main.tf deleted file mode 100644 index df3c5c36b0..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/main.tf +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "dashboard", ghpc_role = "monitoring" }) -} - -locals { - dash_path = "${path.module}/dashboards/${var.base_dashboard}.json.tpl" -} - -resource "google_monitoring_dashboard" "dashboard" { - dashboard_json = templatefile(local.dash_path, { - widgets = var.widgets - deployment_name = var.deployment_name - title = var.title - labels = local.labels - } - ) - project = var.project_id -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/metadata.yaml deleted file mode 100644 index de1a10f57d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - stackdriver.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/outputs.tf deleted file mode 100644 index b7ff35fb0e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/outputs.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "instructions" { - description = "Instructions for accessing the monitoring dashboard" - value = <<-EOT - A monitoring dashboard has been created. To view, navigate to the following URL: - https://console.cloud.google.com/monitoring/dashboards/builder${regex("/[0-9a-z-]*$", google_monitoring_dashboard.dashboard.id)} - EOT -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/variables.tf deleted file mode 100644 index 8194f8b73a..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/variables.tf +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "base_dashboard" { - description = "Baseline dashboard template, select from HPC or Empty" - type = string - default = "HPC" - validation { - condition = contains(["HPC", "Empty"], var.base_dashboard) - error_message = "Must set var.base_dashboard to either \"HPC\" or \"Empty\"." - } -} - -variable "title" { - description = "Title of the created dashboard" - type = string - default = "Cluster Toolkit Dashboard" -} - -variable "widgets" { - description = "List of additional widgets to add to the base dashboard." - type = list(string) - default = [] -} - -variable "labels" { - description = "Labels to add to the monitoring dashboard instance. Key-value pairs." - type = map(string) -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/versions.tf deleted file mode 100644 index d8c4587051..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/monitoring/dashboard/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:dashboard/v1.57.0" - } - - required_version = ">= 0.14.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/README.md b/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/README.md deleted file mode 100644 index b17058daf4..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/README.md +++ /dev/null @@ -1,107 +0,0 @@ -## Description - -This module facilitates the creation of custom firewall rules for existing -networks. - -## Example usage - -This module can be used by other Toolkit modules to create application-specific -firewall rules or in conjunction with the [pre-existing-vpc] module to enable -traffic in existing networks. The snippet below is drawn from the -[ml-slurm.yaml] example: - -```yaml -- group: primary - modules: - - id: network - source: modules/network/pre-existing-vpc - - # this example anticipates that the VPC default network has internal traffic - # allowed and IAP tunneling for SSH connections - - id: firewall_rule - source: modules/network/firewall-rules - use: - - network - settings: - ingress_rules: - - name: $(vars.deployment_name)-allow-internal-traffic - description: Allow internal traffic - destination_ranges: - - $(network.subnetwork_address) - source_ranges: - - $(network.subnetwork_address) - allow: - - protocol: tcp - ports: - - 0-65535 - - protocol: udp - ports: - - 0-65535 - - protocol: icmp - - name: $(vars.deployment_name)-allow-iap-ssh - description: Allow IAP-tunneled SSH connections - destination_ranges: - - $(network.subnetwork_address) - source_ranges: - - 35.235.240.0/20 - allow: - - protocol: tcp - ports: - - 22 -``` - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [firewall\_rule](#module\_firewall\_rule) | terraform-google-modules/network/google//modules/firewall-rules | ~> 9.0 | - -## Resources - -| Name | Type | -|------|------| -| [google_compute_subnetwork.subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [egress\_rules](#input\_egress\_rules) | List of egress rules | 
list(object({
    name                    = string
    description             = optional(string, null)
    disabled                = optional(bool, null)
    priority                = optional(number, null)
    destination_ranges      = optional(list(string), [])
    source_ranges           = optional(list(string), [])
    source_tags             = optional(list(string))
    source_service_accounts = optional(list(string))
    target_tags             = optional(list(string))
    target_service_accounts = optional(list(string))

    allow = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    deny = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    log_config = optional(object({
      metadata = string
    }))
  }))
| `[]` | no | -| [ingress\_rules](#input\_ingress\_rules) | List of ingress rules | 
list(object({
    name                    = string
    description             = optional(string, null)
    disabled                = optional(bool, null)
    priority                = optional(number, null)
    destination_ranges      = optional(list(string), [])
    source_ranges           = optional(list(string), [])
    source_tags             = optional(list(string))
    source_service_accounts = optional(list(string))
    target_tags             = optional(list(string))
    target_service_accounts = optional(list(string))

    allow = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    deny = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    log_config = optional(object({
      metadata = string
    }))
  }))
| `[]` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork whose global network firewall rules will be modified. | `string` | n/a | yes | - -## Outputs - -No outputs. - - -[pre-existing-vpc]: ../pre-existing-vpc/README.md -[ml-slurm.yaml]: ../../../examples/ml-slurm.yaml diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/main.tf b/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/main.tf deleted file mode 100644 index f3f24e78d3..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/main.tf +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# the google_compute_network data source does not allow identification by -# self_link, which uniquely identifies subnet, project, and network -data "google_compute_subnetwork" "subnetwork" { - self_link = var.subnetwork_self_link -} - -# Module-level check for Private Google Access on the subnetwork -check "private_google_access_enabled_subnetwork" { - assert { - condition = data.google_compute_subnetwork.subnetwork.private_ip_google_access - error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.subnetwork.name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." - } -} - -module "firewall_rule" { - source = "terraform-google-modules/network/google//modules/firewall-rules" - version = "~> 9.0" - project_id = data.google_compute_subnetwork.subnetwork.project - network_name = data.google_compute_subnetwork.subnetwork.network - - ingress_rules = var.ingress_rules - egress_rules = var.egress_rules -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/variables.tf deleted file mode 100644 index e7bea8322e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/variables.tf +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork whose global network firewall rules will be modified." - type = string -} - -variable "ingress_rules" { - description = "List of ingress rules" - default = [] - type = list(object({ - name = string - description = optional(string, null) - disabled = optional(bool, null) - priority = optional(number, null) - destination_ranges = optional(list(string), []) - source_ranges = optional(list(string), []) - source_tags = optional(list(string)) - source_service_accounts = optional(list(string)) - target_tags = optional(list(string)) - target_service_accounts = optional(list(string)) - - allow = optional(list(object({ - protocol = string - ports = optional(list(string)) - })), []) - deny = optional(list(object({ - protocol = string - ports = optional(list(string)) - })), []) - log_config = optional(object({ - metadata = string - })) - })) -} - -variable "egress_rules" { - description = "List of egress rules" - default = [] - type = list(object({ - name = string - description = optional(string, null) - disabled = optional(bool, null) - priority = optional(number, null) - destination_ranges = optional(list(string), []) - source_ranges = optional(list(string), []) - source_tags = optional(list(string)) - source_service_accounts = optional(list(string)) - target_tags = optional(list(string)) - target_service_accounts = optional(list(string)) - - allow = optional(list(object({ - protocol = string - ports = optional(list(string)) - })), []) - deny = optional(list(object({ - protocol = string - ports = optional(list(string)) - })), []) - log_config = optional(object({ - metadata = string - })) - })) -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/versions.tf deleted file mode 100644 index c892bc0435..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/firewall-rules/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:firewall-rules/v1.57.0" - } - - required_version = ">= 1.5" -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/README.md b/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/README.md deleted file mode 100644 index 24dc606c88..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/README.md +++ /dev/null @@ -1,143 +0,0 @@ -## Description - -This module accomplishes the following: - -* Creates one [VPC network][cft-network] - * Each VPC contains a variable number of subnetworks as specified in the - `subnetworks_template` variable - * Each subnetwork contains distinct IP address ranges -* Outputs the following unique parameters - * `subnetwork_interfaces` which is compatible with Slurm and vm-instance - modules - * `subnetwork_interfaces_gke` which is compatible with GKE modules - -This module is a simplified version of the VPC module and its main difference -is the variable `subnetwork_template` which is the template for all subnetworks -created within the network. This template contains the following values: - -1. `count`: The number of subnetworks to be created -1. `name_prefix`: The prefix for the subnetwork names -1. `ip_range`: [CIDR-formatted IP range][cidr] -1. `region`: The region where the subnetwork will be deployed - -> [!WARNING] -> The `ip_range` should be always be large enough to split into `count` -> subnetworks and the number of required connections within. - -[cft-network]: https://github.com/terraform-google-modules/terraform-google-network/tree/v10.0.0 -[cidr]: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation - -### Example - -This snippet uses the gpu-vpc module to create a new VPC network named -`test-rdma-net` with 8 subnetworks named `test-mrdma-sub-#` where # ranges from -0 to 7. The subnetworks will split the `ip_range` evenly, starting from bit 16 -(0 indexed). The networks are ingested by the Slurm nodeset within the -`additional_networks` setting. - -```yaml - - id: rdma-net - source: modules/network/gpu-rdma-vpc - settings: - network_name: test-rdma-net - network_profile: https://www.googleapis.com/compute/beta/projects/$(vars.project_id)/global/networkProfiles/$(vars.zone)-vpc-roce - network_routing_mode: REGIONAL - subnetworks_template: - name_prefix: test-mrdma-sub - count: 8 - ip_range: 192.168.0.0/16 - region: $(vars.region) - - - id: a3_nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: [network0] - settings: - machine_type: a3-ultragpu-8g - additional_networks: - $(concat( - [{ - network=null, - subnetwork=network1.subnetwork_self_link, - subnetwork_project=vars.project_id, - nic_type="GVNIC", - queue_count=null, - network_ip="", - stack_type=null, - access_config=[], - ipv6_access_config=[], - alias_ip_range=[] - }], - rdma-net.subnetwork_interfaces - )) - ... -``` - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [vpc](#module\_vpc) | terraform-google-modules/network/google | ~> 10.0 | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | DEPRECATED: enable\_internal\_traffic can not be specified for gpu-rdma-vpc. | `bool` | `null` | no | -| [firewall\_log\_config](#input\_firewall\_log\_config) | DEPRECATED: firewall\_log\_config can not be specified for gpu-rdma-vpc. | `string` | `null` | no | -| [firewall\_rules](#input\_firewall\_rules) | DEPRECATED: firewall\_rules can not be specified for gpu-rdma-vpc. | `any` | `null` | no | -| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | -| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | -| [network\_name](#input\_network\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-net") | `string` | `null` | no | -| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}} | `string` | n/a | yes | -| [network\_routing\_mode](#input\_network\_routing\_mode) | The network routing mode (default "REGIONAL") | `string` | `"REGIONAL"` | no | -| [nic\_type](#input\_nic\_type) | NIC type for use in modules that use the output | `string` | `"MRDMA"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | -| [shared\_vpc\_host](#input\_shared\_vpc\_host) | Makes this project a Shared VPC host if 'true' (default 'false') | `bool` | `false` | no | -| [subnetworks\_template](#input\_subnetworks\_template) | Specifications for the subnetworks that will be created within this VPC.

count (number, required, number of subnets to create, default is 8)
name\_prefix (string, required, subnet name prefix, default is deployment name)
ip\_range (string, required, range of IPs for all subnets to share (CIDR format), default is 192.168.0.0/16)
region (string, optional, region to deploy subnets to, defaults to vars.region) | 
object({
    count       = number
    name_prefix = string
    ip_range    = string
    region      = optional(string)
  })
| 
{
  "count": 8,
  "ip_range": "192.168.0.0/16",
  "name_prefix": null,
  "region": null
}
| no | - -## Outputs - -| Name | Description | -|------|-------------| -| [network\_id](#output\_network\_id) | ID of the new VPC network | -| [network\_name](#output\_network\_name) | Name of the new VPC network | -| [network\_self\_link](#output\_network\_self\_link) | Self link of the new VPC network | -| [subnetwork\_interfaces](#output\_subnetwork\_interfaces) | Full list of subnetwork objects belonging to the new VPC network (compatible with vm-instance and Slurm modules) | -| [subnetwork\_interfaces\_gke](#output\_subnetwork\_interfaces\_gke) | Full list of subnetwork objects belonging to the new VPC network (compatible with gke-node-pool) | -| [subnetwork\_name\_prefix](#output\_subnetwork\_name\_prefix) | Prefix of the RDMA subnetwork names | -| [subnetworks](#output\_subnetworks) | Full list of subnetwork objects belonging to the new VPC network | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/main.tf b/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/main.tf deleted file mode 100644 index 7f2a141db5..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/main.tf +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - autoname = replace(var.deployment_name, "_", "-") - network_name = var.network_name == null ? "${local.autoname}-net" : var.network_name - subnet_prefix = var.subnetworks_template.name_prefix == null ? "${local.autoname}-subnet" : var.subnetworks_template.name_prefix - - new_bits = ceil(log(var.subnetworks_template.count, 2)) - template_subnetworks = [for i in range(var.subnetworks_template.count) : - { - subnet_name = "${local.subnet_prefix}-${i}" - subnet_region = try(var.subnetworks_template.region, var.region) - subnet_ip = cidrsubnet(var.subnetworks_template.ip_range, local.new_bits, i) - } - ] - - firewall_rules = [] - - output_subnets = [ - for subnet in module.vpc.subnets : { - network = null - subnetwork = subnet.self_link - subnetwork_project = null # will populate from subnetwork_self_link - network_ip = null - nic_type = var.nic_type - stack_type = null - queue_count = null - access_config = [] - ipv6_access_config = [] - alias_ip_range = [] - } - ] - - output_subnets_gke = [ - for i in range(length(module.vpc.subnets)) : { - network = local.network_name - subnetwork = local.template_subnetworks[i].subnet_name - subnetwork_project = var.project_id - network_ip = null - nic_type = var.nic_type - stack_type = null - queue_count = null - access_config = [] - ipv6_access_config = [] - alias_ip_range = [] - } - ] -} - -module "vpc" { - source = "terraform-google-modules/network/google" - version = "~> 10.0" - - network_name = local.network_name - project_id = var.project_id - auto_create_subnetworks = false - subnets = local.template_subnetworks - routing_mode = var.network_routing_mode - mtu = var.mtu - description = var.network_description - shared_vpc_host = var.shared_vpc_host - delete_default_internet_gateway_routes = var.delete_default_internet_gateway_routes - firewall_rules = local.firewall_rules - network_profile = var.network_profile -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf deleted file mode 100644 index 0a21f1d3f2..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/outputs.tf +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "network_name" { - description = "Name of the new VPC network" - value = module.vpc.network_name - depends_on = [module.vpc] -} - -output "network_id" { - description = "ID of the new VPC network" - value = module.vpc.network_id - depends_on = [module.vpc] -} - -output "network_self_link" { - description = "Self link of the new VPC network" - value = module.vpc.network_self_link - depends_on = [module.vpc] -} - -output "subnetworks" { - description = "Full list of subnetwork objects belonging to the new VPC network" - value = module.vpc.subnets - depends_on = [module.vpc] -} - -output "subnetwork_interfaces" { - description = "Full list of subnetwork objects belonging to the new VPC network (compatible with vm-instance and Slurm modules)" - value = local.output_subnets - depends_on = [module.vpc] -} - -# The output subnetwork_interfaces is compatible with vm-instance module but not with gke-node-pool -# See https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/99493df21cecf6a092c45298bf7a45e0343cf622/modules/compute/vm-instance/variables.tf#L220 -# So, we need a separate output that makes the network and subnetwork names available -output "subnetwork_interfaces_gke" { - description = "Full list of subnetwork objects belonging to the new VPC network (compatible with gke-node-pool)" - value = local.output_subnets_gke - depends_on = [module.vpc] -} - -output "subnetwork_name_prefix" { - description = "Prefix of the RDMA subnetwork names" - value = var.subnetworks_template.name_prefix -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf deleted file mode 100644 index a30fb50e7d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/variables.tf +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "network_name" { - description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-net\")" - type = string - default = null -} - -variable "region" { - description = "The default region for Cloud resources" - type = string -} - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "mtu" { - type = number - description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." - default = 8896 -} - -variable "subnetworks_template" { - description = <<-EOT - Specifications for the subnetworks that will be created within this VPC. - - count (number, required, number of subnets to create, default is 8) - name_prefix (string, required, subnet name prefix, default is deployment name) - ip_range (string, required, range of IPs for all subnets to share (CIDR format), default is 192.168.0.0/16) - region (string, optional, region to deploy subnets to, defaults to vars.region) - EOT - nullable = false - type = object({ - count = number - name_prefix = string - ip_range = string - region = optional(string) - }) - default = { - count = 8 - name_prefix = null - ip_range = "192.168.0.0/16" - region = null - } - - validation { - condition = var.subnetworks_template.count > 0 - error_message = "Number of subnetworks must be greater than 0" - } - - validation { - condition = can(cidrhost(var.subnetworks_template.ip_range, 0)) - error_message = "IP address range must be in CIDR format." - } -} - -variable "network_routing_mode" { - type = string - default = "REGIONAL" - description = "The network routing mode (default \"REGIONAL\")" - - validation { - condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) - error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." - } -} - -variable "network_description" { - type = string - description = "An optional description of this resource (changes will trigger resource destroy/create)" - default = "" -} - -variable "shared_vpc_host" { - type = bool - description = "Makes this project a Shared VPC host if 'true' (default 'false')" - default = false -} - -variable "delete_default_internet_gateway_routes" { - type = bool - description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" - default = false -} - -variable "enable_internal_traffic" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: enable_internal_traffic can not be specified for gpu-rdma-vpc." - type = bool - default = null - validation { - condition = var.enable_internal_traffic == null - error_message = "DEPRECATED: enable_internal_traffic can not be specified for gpu-rdma-vpc." - } -} - -variable "firewall_rules" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: firewall_rules can not be specified for gpu-rdma-vpc." - type = any - default = null - validation { - condition = var.firewall_rules == null - error_message = "DEPRECATED: firewall_rules can not be specified for gpu-rdma-vpc." - } -} - -variable "firewall_log_config" { # tflint-ignore: terraform_unused_declarations - description = "DEPRECATED: firewall_log_config can not be specified for gpu-rdma-vpc." - type = string - default = null - validation { - condition = var.firewall_log_config == null - error_message = "DEPRECATED: firewall_log_config can not be specified for gpu-rdma-vpc." - } -} - -variable "network_profile" { - description = <<-EOT - A full or partial URL of the network profile to apply to this network. - This field can be set only at resource creation time. For example, the - following are valid URLs: - - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} - - projects/{projectId}/global/networkProfiles/{network_profile_name}} - EOT - type = string - nullable = false - - validation { - condition = can(coalesce(var.network_profile)) - error_message = "var.network_profile must be specified and not an empty string" - } -} - -variable "nic_type" { - description = "NIC type for use in modules that use the output" - type = string - nullable = true - default = "MRDMA" - - validation { - condition = contains(["MRDMA"], var.nic_type) - error_message = "The nic_type must be \"MRDMA\"." - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf deleted file mode 100644 index 71b7106734..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/gpu-rdma-vpc/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 0.15.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/multivpc/README.md b/vdi-test-scott/primary/modules/embedded/modules/network/multivpc/README.md deleted file mode 100644 index 973e6b32c9..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/multivpc/README.md +++ /dev/null @@ -1,136 +0,0 @@ -## Description - -This module accomplishes the following: - -* Creates 2 to 8 [VPC networks][vpc] - * Each VPC contains exactly 1 subnetwork - * Each subnetwork contains distinct IP address ranges -* Outputs the `additional_networks` parameter, which is compatible with Slurm - modules - -There are 4 variables that differentiate this module from the standard VPC -module. - -1. `network_prefix`: The name prefix of the VPCs to be created. All - networks and subnetworks will start with this and end with a unique number. -1. `network_count`: The number of VPCs to be created. -1. `global_ip_address_range`: [CIDR-formatted IP range][cidr] -1. `network_cidr_suffix`: The CIDR suffix that defines the address - space that the individual VPCs will cover. - -> [!WARNING] -> The `network_cidr_suffix` should be always be larger than the CIDR suffix on -> `global_ip_address_range`. The difference between these two suffixes should -> be large enough to accommodate the number of VPCs that are being deployed -> (e.g. CIDR suffix bit difference <= `ceil(log2(network_count)))`). - -> [!NOTE] -> For deployments that need multiple VPCs that do not meet this use-case, users -> should deploy multiple individual VPC modules. - -[vpc]: ../vpc/README.md -[cidr]: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation - -### Example - -This snippet uses the multivpc module to create 8 new VPC networks named -`multivpc-net-#` where # ranges from 0 to 7. Additionally, it creates 1 -subnetwork in each VPC. - -```yaml - - id: network - source: modules/network/vpc - - - id: multinetwork - source: modules/network/multivpc - settings: - network_name_prefix: multivpc-net - network_count: 8 - global_ip_address_range: 172.16.0.0/12 - subnetwork_cidr_suffix: 16 - - - id: a3_nodeset - source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset - use: [network, multinetwork] - settings: - machine_type: a3-highgpu-8g - ... -``` - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.4.0 | - -## Providers - -| Name | Version | -|------|---------| -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [vpcs](#module\_vpcs) | ../vpc | n/a | - -## Resources - -| Name | Type | -|------|------| -| [terraform_data.global_ip_cidr_suffix](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allowed\_ssh\_ip\_ranges](#input\_allowed\_ssh\_ip\_ranges) | A list of CIDR IP ranges from which to allow ssh access | `list(string)` | `[]` | no | -| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [enable\_iap\_rdp\_ingress](#input\_enable\_iap\_rdp\_ingress) | Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels | `bool` | `false` | no | -| [enable\_iap\_ssh\_ingress](#input\_enable\_iap\_ssh\_ingress) | Enable a firewall rule to allow SSH access using IAP tunnels | `bool` | `true` | no | -| [enable\_iap\_winrm\_ingress](#input\_enable\_iap\_winrm\_ingress) | Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels | `bool` | `false` | no | -| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network | `bool` | `true` | no | -| [extra\_iap\_ports](#input\_extra\_iap\_ports) | A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable\_iap variables for standard ports) | `list(string)` | `[]` | no | -| [firewall\_rules](#input\_firewall\_rules) | List of firewall rules | `any` | `[]` | no | -| [global\_ip\_address\_range](#input\_global\_ip\_address\_range) | IP address range (CIDR) that will span entire set of VPC networks | `string` | `"172.16.0.0/12"` | no | -| [ips\_per\_nat](#input\_ips\_per\_nat) | The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT) | `number` | `2` | no | -| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | -| [network\_count](#input\_network\_count) | The number of vpc nettworks to create | `number` | `4` | no | -| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | -| [network\_interface\_defaults](#input\_network\_interface\_defaults) | The template of the network settings to be used on all vpcs. | 
object({
    network            = optional(string)
    subnetwork         = optional(string)
    subnetwork_project = optional(string)
    network_ip         = optional(string, "")
    nic_type           = optional(string, "GVNIC")
    stack_type         = optional(string, "IPV4_ONLY")
    queue_count        = optional(string)
    access_config = optional(list(object({
      nat_ip                 = string
      network_tier           = string
      public_ptr_domain_name = string
    })), [])
    ipv6_access_config = optional(list(object({
      network_tier           = string
      public_ptr_domain_name = string
    })), [])
    alias_ip_range = optional(list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    })), [])
  })
| 
{
  "access_config": [],
  "alias_ip_range": [],
  "ipv6_access_config": [],
  "network": null,
  "network_ip": "",
  "nic_type": "GVNIC",
  "queue_count": null,
  "stack_type": "IPV4_ONLY",
  "subnetwork": null,
  "subnetwork_project": null
}
| no | -| [network\_name\_prefix](#input\_network\_name\_prefix) | The base name of the vpcs and their subnets, will be appended with a sequence number | `string` | `""` | no | -| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}}
When using a Mellanox network profile (contains 'roce'), if firewall\_rules is specified or enable\_internal\_traffic is true, an error will be thrown | `string` | `null` | no | -| [network\_routing\_mode](#input\_network\_routing\_mode) | The network dynamic routing mode | `string` | `"REGIONAL"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | -| [subnetwork\_cidr\_suffix](#input\_subnetwork\_cidr\_suffix) | The size, in CIDR suffix notation, for each network (e.g. 24 for 172.16.0.0/24); changing this will destroy every network. | `number` | `16` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [additional\_networks](#output\_additional\_networks) | Network interfaces for each subnetwork created by this module | -| [network\_ids](#output\_network\_ids) | IDs of the new VPC network | -| [network\_names](#output\_network\_names) | Names of the new VPC networks | -| [network\_self\_links](#output\_network\_self\_links) | Self link of the new VPC network | -| [subnetwork\_addresses](#output\_subnetwork\_addresses) | IP address range of the primary subnetwork | -| [subnetwork\_names](#output\_subnetwork\_names) | Names of the subnetwork created in each network | -| [subnetwork\_self\_links](#output\_subnetwork\_self\_links) | Self link of the primary subnetwork | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/multivpc/main.tf b/vdi-test-scott/primary/modules/embedded/modules/network/multivpc/main.tf deleted file mode 100644 index ad06e793c1..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/multivpc/main.tf +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -locals { - # this input variable is validated to be in CIDR format - network_name = coalesce(replace(var.network_name_prefix, "_", "-"), replace(var.deployment_name, "_", "-")) - global_ip_cidr_prefix = split("/", var.global_ip_address_range)[0] - global_ip_cidr_suffix = split("/", var.global_ip_address_range)[1] - global_ip_cidr_valid = "${local.global_ip_cidr_prefix}/${terraform_data.global_ip_cidr_suffix.output}" - subnetwork_new_bits = var.subnetwork_cidr_suffix - local.global_ip_cidr_suffix - maximum_subnetworks = pow(2, local.subnetwork_new_bits) - additional_networks = [ - for vpc in module.vpcs : - merge(var.network_interface_defaults, { - network = vpc.network_name - subnetwork = vpc.subnetwork_name - subnetwork_project = var.project_id - }) - ] -} - -resource "terraform_data" "global_ip_cidr_suffix" { - input = local.global_ip_cidr_suffix - lifecycle { - precondition { - condition = local.maximum_subnetworks >= var.network_count - error_message = < 1 - error_message = "The minimum VPCs able to be created by this module is 2. Use the standard Toolkit module at modules/network/vpc for count = 1" - } - validation { - condition = var.network_count <= 8 - error_message = "The maximum VPCs able to be created by this module is 8" - } -} - -variable "global_ip_address_range" { - description = "IP address range (CIDR) that will span entire set of VPC networks" - type = string - default = "172.16.0.0/12" - - validation { - condition = can(cidrhost(var.global_ip_address_range, 0)) - error_message = "var.global_ip_address_range must be an IPv4 CIDR range (e.g. \"172.16.0.0/12\")." - } -} - -variable "subnetwork_cidr_suffix" { - description = "The size, in CIDR suffix notation, for each network (e.g. 24 for 172.16.0.0/24); changing this will destroy every network." - type = number - default = 16 -} - -variable "mtu" { - type = number - description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." - default = 8896 -} - -variable "network_routing_mode" { - type = string - default = "REGIONAL" - description = "The network dynamic routing mode" - - validation { - condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) - error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." - } -} - -variable "network_description" { - type = string - description = "An optional description of this resource (changes will trigger resource destroy/create)" - default = "" -} - -variable "ips_per_nat" { - type = number - description = "The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT)" - default = 2 -} - -variable "delete_default_internet_gateway_routes" { - type = bool - description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" - default = false -} - -variable "enable_iap_ssh_ingress" { - type = bool - description = "Enable a firewall rule to allow SSH access using IAP tunnels" - default = true -} - -variable "enable_iap_rdp_ingress" { - type = bool - description = "Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels" - default = false -} - -variable "enable_iap_winrm_ingress" { - type = bool - description = "Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels" - default = false -} - -variable "enable_internal_traffic" { - type = bool - description = "Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network" - default = true -} - -variable "extra_iap_ports" { - type = list(string) - description = "A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable_iap variables for standard ports)" - default = [] -} - -variable "allowed_ssh_ip_ranges" { - type = list(string) - description = "A list of CIDR IP ranges from which to allow ssh access" - default = [] - - validation { - condition = alltrue([for r in var.allowed_ssh_ip_ranges : can(cidrhost(r, 32))]) - error_message = "Each element of var.allowed_ssh_ip_ranges must be a valid CIDR-formatted IPv4 range." - } -} - -variable "firewall_rules" { - type = any - description = "List of firewall rules" - default = [] -} - -variable "network_interface_defaults" { - type = object({ - network = optional(string) - subnetwork = optional(string) - subnetwork_project = optional(string) - network_ip = optional(string, "") - nic_type = optional(string, "GVNIC") - stack_type = optional(string, "IPV4_ONLY") - queue_count = optional(string) - access_config = optional(list(object({ - nat_ip = string - network_tier = string - public_ptr_domain_name = string - })), []) - ipv6_access_config = optional(list(object({ - network_tier = string - public_ptr_domain_name = string - })), []) - alias_ip_range = optional(list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })), []) - }) - description = "The template of the network settings to be used on all vpcs." - default = { - network = null - subnetwork = null - subnetwork_project = null - network_ip = "" - nic_type = "GVNIC" - stack_type = "IPV4_ONLY" - queue_count = null - access_config = [] - ipv6_access_config = [] - alias_ip_range = [] - } -} - -variable "network_profile" { - type = string - description = <<-EOT - A full or partial URL of the network profile to apply to this network. - This field can be set only at resource creation time. For example, the - following are valid URLs: - - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} - - projects/{projectId}/global/networkProfiles/{network_profile_name}} - When using a Mellanox network profile (contains 'roce'), if firewall_rules is specified or enable_internal_traffic is true, an error will be thrown - EOT - default = null -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/multivpc/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/network/multivpc/versions.tf deleted file mode 100644 index e75a67f7b6..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/multivpc/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 1.4.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/README.md b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/README.md deleted file mode 100644 index 4d63b17091..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/README.md +++ /dev/null @@ -1,94 +0,0 @@ -## Description - -This module discovers a subnetwork that already exists in Google Cloud and -outputs subnetwork attributes that uniquely identify it for use by other modules. - -For example, the blueprint below discovers the referred to subnetwork. -With the `use` keyword, the [vm-instance] module accepts the `subnetwork_self_link` -input variables that uniquely identify the subnetwork in which the VM will be created. - -[vpc]: ../vpc/README.md -[vm-instance]: ../../compute/vm-instance/README.md - -> **_NOTE:_** Additional IAM work is needed for this to work correctly. - -### Example - -```yaml -- id: network - source: modules/network/pre-existing-subnetwork - settings: - subnetwork_self_link: https://www.googleapis.com/compute/v1/projects/name-of-host-project/regions/REGION/subnetworks/SUBNETNAME - -- id: example_vm - source: modules/compute/vm-instance - use: - - network - settings: - name_prefix: example - machine_type: c2-standard-4 -``` - -As described in documentation: -[https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork] - -If subnetwork_self_link is provided then name,region,project is ignored. - -## License - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_subnetwork.primary_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [project](#input\_project) | Name of the project that owns the subnetwork | `string` | `null` | no | -| [region](#input\_region) | Region in which to search for primary subnetwork | `string` | `null` | no | -| [subnetwork\_name](#input\_subnetwork\_name) | Name of the pre-existing VPC subnetwork | `string` | `null` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | Self-link of the subnet in the VPC | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [subnetwork](#output\_subnetwork) | Full subnetwork object in the primary region | -| [subnetwork\_address](#output\_subnetwork\_address) | Subnetwork IP range in the primary region | -| [subnetwork\_name](#output\_subnetwork\_name) | Name of the subnetwork in the primary region | -| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Subnetwork self-link in the primary region | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/main.tf b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/main.tf deleted file mode 100644 index 9fb206f969..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/main.tf +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - - -data "google_compute_subnetwork" "primary_subnetwork" { - name = var.subnetwork_name - region = var.region - project = var.project - self_link = var.subnetwork_self_link - - lifecycle { - postcondition { - condition = self.self_link != null - error_message = "The subnetwork: ${coalesce(var.subnetwork_name, var.subnetwork_self_link)} could not be found." - } - } -} - -# Module-level check for Private Google Access on the subnetwork -check "private_google_access_enabled_subnetwork" { - assert { - condition = data.google_compute_subnetwork.primary_subnetwork.private_ip_google_access - error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.primary_subnetwork.name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml deleted file mode 100644 index 6a6f1e5757..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com -ghpc: - has_to_be_used: true diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf deleted file mode 100644 index 868708dc6b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/outputs.tf +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "subnetwork" { - description = "Full subnetwork object in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork -} - -output "subnetwork_name" { - description = "Name of the subnetwork in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.name -} - -output "subnetwork_self_link" { - description = "Subnetwork self-link in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.self_link -} - -output "subnetwork_address" { - description = "Subnetwork IP range in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.ip_cidr_range -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf deleted file mode 100644 index d5191843e8..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/variables.tf +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "subnetwork_self_link" { - description = "Self-link of the subnet in the VPC" - type = string - default = null -} - -variable "project" { - description = "Name of the project that owns the subnetwork" - type = string - default = null -} - -variable "subnetwork_name" { - description = "Name of the pre-existing VPC subnetwork" - type = string - default = null -} - -variable "region" { - description = "Region in which to search for primary subnetwork" - type = string - default = null -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf deleted file mode 100644 index e90ae5991d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-subnetwork/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:pre-existing-subnetwork/v1.57.0" - } - - required_version = ">= 1.5" -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/README.md b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/README.md deleted file mode 100644 index 38a1840c2d..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/README.md +++ /dev/null @@ -1,110 +0,0 @@ -## Description - -This module discovers a VPC network that already exists in Google Cloud and -outputs network attributes that uniquely identify it for use by other modules. -The module outputs are aligned with the [vpc module][vpc] so that it can be used -as a drop-in substitute when a VPC already exists. - -For example, the blueprint below discovers the "default" global network and the -"default" regional subnetwork in us-central1. With the `use` keyword, the -[vm-instance] module accepts the `network_self_link` and `subnetwork_self_link` -input variables that uniquely identify the network and subnetwork in which the -VM will be created. - -[vpc]: ../vpc/README.md -[vm-instance]: ../../compute/vm-instance/README.md - -### Example - -```yaml -- id: network1 - source: modules/network/pre-existing-vpc - settings: - project_id: $(vars.project_id) - region: us-central1 - -- id: example_vm - source: modules/compute/vm-instance - use: - - network1 - settings: - name_prefix: example - machine_type: c2-standard-4 -``` - -> **_NOTE:_** The `project_id` and `region` settings would be inferred from the -> deployment variables of the same name, but they are included here for clarity. - -### Use shared-vpc - -If a network is created in different project, this module can be used to -reference the network. To use a network from a different project first make sure -you have a [cloud nat][cloudnat] and [IAP][iap] forwarding. For more details, -refer [shared-vpc][shared-vpc-doc] - -[cloudnat]: https://cloud.google.com/nat/docs/overview -[iap]: https://cloud.google.com/iap/docs/using-tcp-forwarding -[shared-vpc-doc]: ../../../examples/README.md#hpc-slurm-sharedvpcyaml-community-badge-experimental-badge - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_compute_network.vpc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network) | data source | -| [google_compute_subnetwork.primary_subnetwork](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [network\_name](#input\_network\_name) | Name of the existing VPC network | `string` | `"default"` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | Region in which to search for primary subnetwork | `string` | n/a | yes | -| [subnetwork\_name](#input\_subnetwork\_name) | Name of the pre-existing VPC subnetwork; defaults to var.network\_name if set to null. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [network\_id](#output\_network\_id) | ID of the existing VPC network | -| [network\_name](#output\_network\_name) | Name of the existing VPC network | -| [network\_self\_link](#output\_network\_self\_link) | Self link of the existing VPC network | -| [subnetwork](#output\_subnetwork) | Full subnetwork object in the primary region | -| [subnetwork\_address](#output\_subnetwork\_address) | Subnetwork IP range in the primary region | -| [subnetwork\_name](#output\_subnetwork\_name) | Name of the subnetwork in the primary region | -| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Subnetwork self-link in the primary region | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/main.tf b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/main.tf deleted file mode 100644 index ed332bab72..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/main.tf +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - - -data "google_compute_network" "vpc" { - name = var.network_name - project = var.project_id - - lifecycle { - postcondition { - condition = self.self_link != null - error_message = "The network: ${var.network_name} could not be found in project: ${var.project_id}." - } - } -} - -locals { - subnetwork_name = var.subnetwork_name != null ? var.subnetwork_name : var.network_name -} - -data "google_compute_subnetwork" "primary_subnetwork" { - name = local.subnetwork_name - region = var.region - project = var.project_id - - lifecycle { - postcondition { - condition = self.self_link != null - error_message = "The subnetwork: ${local.subnetwork_name} could not be found in project: ${var.project_id} and region: ${var.region}." - } - } -} - -# Module-level check for Private Google Access on the subnetwork -check "private_google_access_enabled_subnetwork" { - assert { - condition = data.google_compute_subnetwork.primary_subnetwork.private_ip_google_access - error_message = "Private Google Access is disabled for subnetwork '${data.google_compute_subnetwork.primary_subnetwork.name}'. This may cause connectivity issues for instances without external IPs trying to access Google APIs and services." - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/outputs.tf deleted file mode 100644 index 00861af5ca..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/outputs.tf +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "network_name" { - description = "Name of the existing VPC network" - value = data.google_compute_network.vpc.name -} - -output "network_id" { - description = "ID of the existing VPC network" - value = data.google_compute_network.vpc.id -} - -output "network_self_link" { - description = "Self link of the existing VPC network" - value = data.google_compute_network.vpc.self_link -} - -output "subnetwork" { - description = "Full subnetwork object in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork -} - -output "subnetwork_name" { - description = "Name of the subnetwork in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.name -} - -output "subnetwork_self_link" { - description = "Subnetwork self-link in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.self_link -} - -output "subnetwork_address" { - description = "Subnetwork IP range in the primary region" - value = data.google_compute_subnetwork.primary_subnetwork.ip_cidr_range -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/variables.tf deleted file mode 100644 index 291a81604a..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/variables.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "network_name" { - description = "Name of the existing VPC network" - type = string - default = "default" -} - -variable "subnetwork_name" { - description = "Name of the pre-existing VPC subnetwork; defaults to var.network_name if set to null." - type = string - default = null -} - -variable "region" { - description = "Region in which to search for primary subnetwork" - type = string -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/versions.tf deleted file mode 100644 index d798ca3f8c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/pre-existing-vpc/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:pre-existing-vpc/v1.57.0" - } - - required_version = ">= 1.5" -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/vpc/README.md b/vdi-test-scott/primary/modules/embedded/modules/network/vpc/README.md deleted file mode 100644 index 7521537a9f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/vpc/README.md +++ /dev/null @@ -1,237 +0,0 @@ -## Description - -This module creates a new [VPC network][vpc] with 1 or more subnetworks and -a [Cloud Router][router] for every region with a subnetwork. By default, it will -create: - -* A [Cloud NAT][nat] to enable outbound access to the public internet for VMs - without public IP addresses; VMs with public IP addresses bypass the NAT to - directly access the public internet -* A firewall rule that enables inbound SSH access from [Identity-Aware - Proxy][iap] -* A firewall rule that enables all traffic internal to the network - -This behavior is optional and can be configured as [described below](#inputs). -This module is based on networking support in the [Cloud Foundation -Toolkit][cft]. We recommend following the [documentation for the network -module][cft-network] and [submodules][cft-network-submodules] for more details. -In particular, the detailed structure of input variables can be found for: - -* [var.firewall\_rules](https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/firewall-rules#inputs) -* [var.secondary\_ranges](https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/subnets#inputs) - -[vpc]: https://cloud.google.com/vpc -[router]: https://github.com/terraform-google-modules/terraform-google-cloud-router -[nat]: https://github.com/terraform-google-modules/terraform-google-cloud-nat -[iap]: https://cloud.google.com/iap -[cft]: https://cloud.google.com/foundation-toolkit -[cft-network]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0 -[cft-network-submodules]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules - -Additionally, [Google Private Access][gpa] is enabled by default on all -subnetworks unless it is explicitly disabled. This setting ensures that all VMs -can use Google services such as [Cloud Storage][gcs] even if they do not have -public IP addresses or Cloud NAT is disabled. - -[gpa]: https://cloud.google.com/vpc/docs/private-google-access -[gcs]: https://cloud.google.com/storage - -### Example - -This creates a new VPC network named `cluster-net`. - -```yaml - - id: network1 - source: modules/network/vpc - settings: - network_name: cluster-net -``` - -### Deprecation warning - -The variables listed below have been deprecated and will be removed in a future -release. Until they are removed,You may continue to use them in Toolkit -blueprints with the same functionality as documented in the [Toolkit 1.0 -release][vpc1.0]. - -* Deprecated variables - * `var.primary_subnetwork` - * `var.additional_subnetworks` - * `var.subnetwork_size` - -[vpc1.0]: https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/v1.0.0/modules/network/vpc/README.md - -The following variables have been added to support explicit IP ranges for -subnetworks while retaining existing functionality. We advise adopting them even -if not using explicit IP ranges . The Toolkit ***does not support*** mixing -deprecated variables with the new replacements. The new functionality is -described in [more detail below](#subnetworks). - -* New variables to adopt - * `var.subnetworks` - * A value for this can be generated by merging `var.primary_subnetwork` and - `var.additional_subnetworks` into a single list - * `var.default_primary_subnetwork_size` - * This variable has been renamed for clarity; its value can be directly - copied from an explicit setting for `var.subnetwork_size`; if your blueprint - does not have an explicit setting, the default values are the same - -### Subnetworks - -This module will always provision at least 1 "primary" subnetwork in which most -resources are expected to be provisioned. This primary subnetwork is determined -by - -1. The first element of [var.subnetworks](#input_subnetworks) if it is not the - empty list -2. A default subnetwork automatically calculated from - * [var.subnetwork_name](#input_subnetwork_name) - * [var.region](#input_region) - * [var.network_address_range](#input_network_address_range) - * [var.default_primary_subnetwork_size](#input_default_primary_subnetwork_size) - -If `var.subnetworks` is provided then the primary subnetwork name is taken -explicitly from it and `var.subnetwork_name` is ignored. - -`var.subnetworks` behaves identically to the [Cloud Foundation Toolkit subnets -module][cftsubnets] with the lone exception that one can provide ***one*** of -the following settings for each subnetwork: - -* `new_bits` -* `subnet_ip` - -If each subnetwork defines `subnet_ip` then these are taken to be their explicit -CIDR IP ranges. If each subnetwork defines `new_bits`, then these are taken to -be the size of the CIDR subnetwork (in bits). IP ranges for each subnetwork are -calculated using `var.network_address_range` as the base IP, producing the most -compact set of subnetworks possible. - -> **_NOTE:_** we do not presently support the modification of individual subnetworks -> when using this module to provision more than 1 subnetwork using automatically -> calculated IP ranges based upon `new_bits`. Doing so will cause IP ranges to be -> recalculated for each subnetwork. We advise appending new subnetworks to the end -> of `var.subnetworks`. - -[cftsubnets]: https://github.com/terraform-google-modules/terraform-google-network/tree/v5.1.0/modules/subnets - -### SSH Access - -By default a firewall rule is created to allow inbound SSH access from -[Identity-Aware Proxy][iap]. A user must have the `IAP-Secured Tunnel User` -(`roles/iap.tunnelResourceAccessor`) IAM role to be able to SSH over IAP. - -To allow regular SSH access from a known IP address you can add the following -`firewall_rules` setting to the `vpc` module: - -```yaml - - id: network1 - source: modules/network/vpc - settings: - firewall_rules: - - name: ssh-my-machine - direction: INGRESS - ranges: [/32] - allow: - - protocol: tcp - ports: [22] -``` - -> **Note**: You must populate the above example with the source IP address from -> which you plan to SSH from. You can use a service like -> [whatismyip.com](https://whatismyip.com) to determine your IP address. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.15.0 | - -## Providers - -| Name | Version | -|------|---------| -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [cloud\_router](#module\_cloud\_router) | terraform-google-modules/cloud-router/google | ~> 6.0 | -| [nat\_ip\_addresses](#module\_nat\_ip\_addresses) | terraform-google-modules/address/google | ~> 4.1 | -| [vpc](#module\_vpc) | terraform-google-modules/network/google | ~> 10.0 | - -## Resources - -| Name | Type | -|------|------| -| [terraform_data.cloud_nat_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [terraform_data.network_profile_firewall_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [terraform_data.secondary_ranges_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_subnetworks](#input\_additional\_subnetworks) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `list(map(string))` | `null` | no | -| [allowed\_ssh\_ip\_ranges](#input\_allowed\_ssh\_ip\_ranges) | A list of CIDR IP ranges from which to allow ssh access | `list(string)` | `[]` | no | -| [default\_primary\_subnetwork\_size](#input\_default\_primary\_subnetwork\_size) | The size, in CIDR bits, of the default primary subnetwork unless explicitly defined in var.subnetworks | `number` | `15` | no | -| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [enable\_cloud\_nat](#input\_enable\_cloud\_nat) | Enable the creation of Cloud NATs. | `bool` | `true` | no | -| [enable\_cloud\_router](#input\_enable\_cloud\_router) | Enable the creation of a Cloud Router for your VPC. For more information on Cloud Routers see https://cloud.google.com/network-connectivity/docs/router/concepts/overview | `bool` | `true` | no | -| [enable\_iap\_rdp\_ingress](#input\_enable\_iap\_rdp\_ingress) | Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels | `bool` | `false` | no | -| [enable\_iap\_ssh\_ingress](#input\_enable\_iap\_ssh\_ingress) | Enable a firewall rule to allow SSH access using IAP tunnels | `bool` | `true` | no | -| [enable\_iap\_winrm\_ingress](#input\_enable\_iap\_winrm\_ingress) | Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels | `bool` | `false` | no | -| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network | `bool` | `true` | no | -| [extra\_iap\_ports](#input\_extra\_iap\_ports) | A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable\_iap variables for standard ports) | `list(string)` | `[]` | no | -| [firewall\_log\_config](#input\_firewall\_log\_config) | Firewall log configuration for Toolkit firewall rules (var.enable\_iap\_ssh\_ingress and others) | `string` | `"DISABLE_LOGGING"` | no | -| [firewall\_rules](#input\_firewall\_rules) | List of firewall rules | `any` | `[]` | no | -| [ips\_per\_nat](#input\_ips\_per\_nat) | The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT) | `number` | `2` | no | -| [labels](#input\_labels) | Labels to add to network resources that support labels. Key-value pairs of strings. | `map(string)` | `{}` | no | -| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | -| [network\_address\_range](#input\_network\_address\_range) | IP address range (CIDR) for global network | `string` | `"10.0.0.0/9"` | no | -| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | -| [network\_name](#input\_network\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-net") | `string` | `null` | no | -| [network\_profile](#input\_network\_profile) | A full or partial URL of the network profile to apply to this network.
This field can be set only at resource creation time. For example, the
following are valid URLs:
- https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name}
- projects/{projectId}/global/networkProfiles/{network\_profile\_name}}
When using a Mellanox network profile (contains 'roce'), if firewall\_rules is specified or enable\_internal\_traffic is true, an error will be thrown | `string` | `null` | no | -| [network\_routing\_mode](#input\_network\_routing\_mode) | The network routing mode (default "GLOBAL") | `string` | `"GLOBAL"` | no | -| [primary\_subnetwork](#input\_primary\_subnetwork) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `map(string)` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | -| [secondary\_ranges](#input\_secondary\_ranges) | "Secondary ranges associated with the subnets.
This will be deprecated in favour of secondary\_ranges\_list at a later date.
Please migrate to using the same." | `map(list(object({ range_name = string, ip_cidr_range = string })))` | `{}` | no | -| [secondary\_ranges\_list](#input\_secondary\_ranges\_list) | "List of secondary ranges associated with the subnetworks.
Each subnetwork must be specified at most once in this list." | 
list(object({
    subnetwork_name = string,
    ranges = list(object({
      range_name    = string,
      ip_cidr_range = string
    }))
  }))
| `[]` | no | -| [shared\_vpc\_host](#input\_shared\_vpc\_host) | Makes this project a Shared VPC host if 'true' (default 'false') | `bool` | `false` | no | -| [subnetwork\_name](#input\_subnetwork\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-primary-subnet") | `string` | `null` | no | -| [subnetwork\_size](#input\_subnetwork\_size) | DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions | `number` | `null` | no | -| [subnetworks](#input\_subnetworks) | List of subnetworks to create within the VPC. If left empty, it will be
replaced by a single, default subnetwork constructed from other parameters
(e.g. var.region). In all cases, the first subnetwork in the list is identified
by outputs as a "primary" subnetwork.

subnet\_name (string, required, name of subnet)
subnet\_region (string, required, region of subnet)
subnet\_ip (string, mutually exclusive with new\_bits, CIDR-formatted IP range for subnetwork)
new\_bits (number, mutually exclusive with subnet\_ip, CIDR bits used to calculate subnetwork range)
subnet\_private\_access (bool, optional, Enable Private Access on subnetwork)
subnet\_flow\_logs (map(string), optional, Configure Flow Logs see terraform-google-network module)
description (string, optional, Description of Network)
purpose (string, optional, related to Load Balancing)
role (string, optional, related to Load Balancing) | `list(map(string))` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [nat\_ips](#output\_nat\_ips) | External IPs of the Cloud NAT from which outbound internet traffic will arrive (empty list if no NAT is used) | -| [network\_id](#output\_network\_id) | ID of the new VPC network | -| [network\_name](#output\_network\_name) | Name of the new VPC network | -| [network\_self\_link](#output\_network\_self\_link) | Self link of the new VPC network | -| [subnetwork](#output\_subnetwork) | Primary subnetwork object | -| [subnetwork\_address](#output\_subnetwork\_address) | IP address range of the primary subnetwork | -| [subnetwork\_name](#output\_subnetwork\_name) | Name of the primary subnetwork | -| [subnetwork\_self\_link](#output\_subnetwork\_self\_link) | Self link of the primary subnetwork | -| [subnetworks](#output\_subnetworks) | Full list of subnetwork objects belonging to the new VPC network | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/vpc/main.tf b/vdi-test-scott/primary/modules/embedded/modules/network/vpc/main.tf deleted file mode 100644 index fecfc73df3..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/vpc/main.tf +++ /dev/null @@ -1,256 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -resource "terraform_data" "secondary_ranges_validation" { - lifecycle { - precondition { - condition = !(length(var.secondary_ranges) > 0 && length(var.secondary_ranges_list) > 0) - error_message = "Only one of var.secondary_ranges or var.secondary_ranges_list should be specified" - } - } -} - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "vpc", ghpc_role = "network" }) -} - -locals { - autoname = replace(var.deployment_name, "_", "-") - network_name = var.network_name == null ? "${local.autoname}-net" : var.network_name - subnetwork_name = var.subnetwork_name == null ? "${local.autoname}-primary-subnet" : var.subnetwork_name - - # define a default subnetwork for cases in which no explicit subnetworks are - # defined in var.subnetworks - default_primary_subnetwork_cidr_block = cidrsubnet(var.network_address_range, var.default_primary_subnetwork_size, 0) - default_primary_subnetwork = { - subnet_name = local.subnetwork_name - subnet_ip = local.default_primary_subnetwork_cidr_block - subnet_region = var.region - subnet_private_access = true - subnet_flow_logs = false - description = "primary subnetwork in ${local.network_name}" - purpose = null - role = null - } - - # Identify user-supplied primary subnetwork - # (1) explicit var.subnetworks[0] - # (2) implicit local default subnetwork - input_primary_subnetwork = coalesce(try(var.subnetworks[0], null), local.default_primary_subnetwork) - - # Identify user-supplied additional subnetworks - # (1) explicit var.subnetworks[1:end] - # (2) empty list - input_additional_subnetworks = try(slice(var.subnetworks, 1, length(var.subnetworks)), []) - - # at this point we have constructed a list of subnetworks but need to extract - # user-provided CIDR blocks or calculate them from user-provided new_bits - # after we complete deprecation, local.all_subnetworks can be replaced with - # var.subnetworks (or local.default_primary_subnetwork if that is null) - input_subnetworks = concat([local.input_primary_subnetwork], local.input_additional_subnetworks) - subnetworks_cidr_blocks = try( - local.input_subnetworks[*]["subnet_ip"], - cidrsubnets(var.network_address_range, local.input_subnetworks[*]["new_bits"]...) - ) - - # merge in the CIDR blocks (even when already there) and remove new_bits - subnetworks = [for i, subnet in local.input_subnetworks : - merge({ for k, v in subnet : k => v if k != "new_bits" }, { "subnet_ip" = local.subnetworks_cidr_blocks[i] }) - ] - - # gather the unique regions for purposes of creating Router/NAT - cloud_router_regions = var.enable_cloud_router ? distinct([for subnet in local.subnetworks : subnet.subnet_region]) : [] - cloud_nat_regions = var.enable_cloud_nat ? local.cloud_router_regions : [] - - # this comprehension should have 1 and only 1 match - output_primary_subnetwork = one([for k, v in module.vpc.subnets : v if k == "${local.subnetworks[0].subnet_region}/${local.subnetworks[0].subnet_name}"]) - output_primary_subnetwork_name = local.output_primary_subnetwork.name - output_primary_subnetwork_self_link = local.output_primary_subnetwork.self_link - output_primary_subnetwork_ip_cidr_range = local.output_primary_subnetwork.ip_cidr_range - - iap_ports = distinct(concat(compact([ - var.enable_iap_rdp_ingress ? "3389" : "", - var.enable_iap_ssh_ingress ? "22" : "", - var.enable_iap_winrm_ingress ? "5986" : "", - ]), var.extra_iap_ports)) - - firewall_log_api_values = { - "DISABLE_LOGGING" = null - "INCLUDE_ALL_METADATA" = { metadata = "INCLUDE_ALL_METADATA" }, - "EXCLUDE_ALL_METADATA" = { metadata = "EXCLUDE_ALL_METADATA" }, - } - firewall_log_config = lookup(local.firewall_log_api_values, var.firewall_log_config, null) - - allow_iap_ingress = { - name = "${local.network_name}-fw-allow-iap-ingress" - description = "allow TCP access via Identity-Aware Proxy" - direction = "INGRESS" - priority = null - ranges = ["35.235.240.0/20"] - source_tags = null - source_service_accounts = null - target_tags = null - target_service_accounts = null - allow = [{ - protocol = "tcp" - ports = local.iap_ports - }] - deny = [] - log_config = local.firewall_log_config - } - - allow_ssh_ingress = { - name = "${local.network_name}-fw-allow-ssh-ingress" - description = "allow SSH access" - direction = "INGRESS" - priority = null - ranges = var.allowed_ssh_ip_ranges - source_tags = null - source_service_accounts = null - target_tags = null - target_service_accounts = null - allow = [{ - protocol = "tcp" - ports = ["22"] - }] - deny = [] - log_config = local.firewall_log_config - } - - allow_internal_traffic = { - name = "${local.network_name}-fw-allow-internal-traffic" - priority = null - description = "allow traffic between nodes of this VPC" - direction = "INGRESS" - ranges = [var.network_address_range] - source_tags = null - source_service_accounts = null - target_tags = null - target_service_accounts = null - allow = [{ - protocol = "tcp" - ports = ["0-65535"] - }, { - protocol = "udp" - ports = ["0-65535"] - }, { - protocol = "icmp" - ports = null - }, - ] - deny = [] - log_config = local.firewall_log_config - } - - firewall_rules = concat( - var.firewall_rules, - length(var.allowed_ssh_ip_ranges) > 0 ? [local.allow_ssh_ingress] : [], - var.enable_internal_traffic ? [local.allow_internal_traffic] : [], - length(local.iap_ports) > 0 ? [local.allow_iap_ingress] : [] - ) - - secondary_ranges_map = { - for secondary_range in var.secondary_ranges_list : - secondary_range.subnetwork_name => secondary_range.ranges - } -} - -resource "terraform_data" "network_profile_firewall_validation" { - lifecycle { - precondition { - condition = !(try(strcontains(var.network_profile, "roce"), false) && length(local.firewall_rules) > 0) - error_message = "If var.network_profile contains 'roce', var.firewall_rules must be empty and var.enable_internal_traffic must be false, please see: https://cloud.google.com/vpc/docs/rdma-network-profiles#additional_features_that_dont_apply_to_traffic_from_rdma_nics" - } - } -} - -module "vpc" { - source = "terraform-google-modules/network/google" - version = "~> 10.0" - - depends_on = [terraform_data.network_profile_firewall_validation] - - network_name = local.network_name - project_id = var.project_id - auto_create_subnetworks = false - subnets = local.subnetworks - secondary_ranges = length(local.secondary_ranges_map) > 0 ? local.secondary_ranges_map : var.secondary_ranges - routing_mode = var.network_routing_mode - mtu = var.mtu - description = var.network_description - shared_vpc_host = var.shared_vpc_host - delete_default_internet_gateway_routes = var.delete_default_internet_gateway_routes - firewall_rules = local.firewall_rules - network_profile = var.network_profile -} - -resource "terraform_data" "cloud_nat_validation" { - lifecycle { - precondition { - condition = var.enable_cloud_router == true || var.enable_cloud_nat == false - error_message = <<-EOD - "Cannot have Cloud NAT without a Cloud Router. If you desire Cloud NAT functionality please set `enable_cloud_router` to true." - EOD - } - } -} - -# This use of the module may appear odd when var.ips_per_nat = 0. The module -# will be called for all regions with subnetworks but names will be set to the -# empty list. This is a perfectly valid value (the default!). In this scenario, -# no IP addresses are created and all module outputs are empty lists. -# -# https://github.com/terraform-google-modules/terraform-google-address/blob/v3.1.1/variables.tf#L27 -# https://github.com/terraform-google-modules/terraform-google-address/blob/v3.1.1/outputs.tf -module "nat_ip_addresses" { - source = "terraform-google-modules/address/google" - version = "~> 4.1" - - depends_on = [terraform_data.cloud_nat_validation] - - for_each = toset(local.cloud_nat_regions) - - project_id = var.project_id - region = each.value - # an external, regional (not global) IP address is suited for a regional NAT - address_type = "EXTERNAL" - global = false - labels = local.labels - names = [for idx in range(var.ips_per_nat) : "${local.network_name}-nat-ips-${each.value}-${idx}"] -} - -module "cloud_router" { - source = "terraform-google-modules/cloud-router/google" - version = "~> 6.0" - - depends_on = [terraform_data.cloud_nat_validation] - - for_each = toset(local.cloud_router_regions) - - project = var.project_id - name = "${local.network_name}-router" - region = each.value - network = module.vpc.network_name - # in scenario with no NAT IPs, no NAT is created even if router is created - # https://github.com/terraform-google-modules/terraform-google-cloud-router/blob/v2.0.0/nat.tf#L18-L20 - nats = length(module.nat_ip_addresses[each.value].self_links) == 0 ? [] : [ - { - name : "cloud-nat-${each.value}", - nat_ips : module.nat_ip_addresses[each.value].self_links - }, - ] -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/vpc/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/network/vpc/metadata.yaml deleted file mode 100644 index 4c2f23a8d7..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/vpc/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/vpc/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/network/vpc/outputs.tf deleted file mode 100644 index c2ee6bdf6b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/vpc/outputs.tf +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -output "network_name" { - description = "Name of the new VPC network" - value = module.vpc.network_name - depends_on = [module.vpc, module.cloud_router] -} - -output "network_id" { - description = "ID of the new VPC network" - value = module.vpc.network_id - depends_on = [module.vpc, module.cloud_router] -} - -output "network_self_link" { - description = "Self link of the new VPC network" - value = module.vpc.network_self_link - depends_on = [module.vpc, module.cloud_router] -} - -output "subnetworks" { - description = "Full list of subnetwork objects belonging to the new VPC network" - value = module.vpc.subnets - depends_on = [module.vpc, module.cloud_router] -} - -output "subnetwork" { - description = "Primary subnetwork object" - value = local.output_primary_subnetwork - depends_on = [module.vpc, module.cloud_router] -} - -output "subnetwork_name" { - description = "Name of the primary subnetwork" - value = local.output_primary_subnetwork_name - depends_on = [module.vpc, module.cloud_router] -} - -output "subnetwork_self_link" { - description = "Self link of the primary subnetwork" - value = local.output_primary_subnetwork_self_link - depends_on = [module.vpc, module.cloud_router] -} - -output "subnetwork_address" { - description = "IP address range of the primary subnetwork" - value = local.output_primary_subnetwork_ip_cidr_range - depends_on = [module.vpc, module.cloud_router] -} - -output "nat_ips" { - description = "External IPs of the Cloud NAT from which outbound internet traffic will arrive (empty list if no NAT is used)" - value = flatten([for ipmod in module.nat_ip_addresses : ipmod.addresses]) -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/vpc/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/network/vpc/variables.tf deleted file mode 100644 index 32a9a00e74..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/vpc/variables.tf +++ /dev/null @@ -1,301 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "labels" { - description = "Labels to add to network resources that support labels. Key-value pairs of strings." - type = map(string) - default = {} - nullable = false -} - -variable "network_name" { - description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-net\")" - type = string - default = null -} - -variable "subnetwork_name" { - description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-primary-subnet\")" - type = string - default = null -} - -# tflint-ignore: terraform_unused_declarations -variable "subnetwork_size" { - description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" - type = number - default = null - validation { - condition = var.subnetwork_size == null - error_message = "subnetwork_size is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." - } -} - -variable "default_primary_subnetwork_size" { - description = "The size, in CIDR bits, of the default primary subnetwork unless explicitly defined in var.subnetworks" - type = number - default = 15 -} - -variable "region" { - description = "The default region for Cloud resources" - type = string -} - -variable "deployment_name" { - description = "The name of the current deployment" - type = string -} - -variable "network_address_range" { - description = "IP address range (CIDR) for global network" - type = string - default = "10.0.0.0/9" - - validation { - condition = can(cidrhost(var.network_address_range, 0)) - error_message = "IP address range must be in CIDR format." - } -} - -variable "mtu" { - type = number - description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." - default = 8896 -} - -variable "subnetworks" { - description = <<-EOT - List of subnetworks to create within the VPC. If left empty, it will be - replaced by a single, default subnetwork constructed from other parameters - (e.g. var.region). In all cases, the first subnetwork in the list is identified - by outputs as a "primary" subnetwork. - - subnet_name (string, required, name of subnet) - subnet_region (string, required, region of subnet) - subnet_ip (string, mutually exclusive with new_bits, CIDR-formatted IP range for subnetwork) - new_bits (number, mutually exclusive with subnet_ip, CIDR bits used to calculate subnetwork range) - subnet_private_access (bool, optional, Enable Private Access on subnetwork) - subnet_flow_logs (map(string), optional, Configure Flow Logs see terraform-google-network module) - description (string, optional, Description of Network) - purpose (string, optional, related to Load Balancing) - role (string, optional, related to Load Balancing) - EOT - type = list(map(string)) - default = [] - validation { - condition = alltrue([ - for s in var.subnetworks : can(s["subnet_name"]) - ]) - error_message = "All subnetworks must define \"subnet_name\"." - } - validation { - condition = alltrue([ - for s in var.subnetworks : can(s["subnet_region"]) - ]) - error_message = "All subnetworks must define \"subnet_region\"." - } - validation { - condition = alltrue([ - for s in var.subnetworks : can(s["subnet_ip"]) != can(s["new_bits"]) - ]) - error_message = "All subnetworks must define exactly one of \"subnet_ip\" or \"new_bits\"." - } - validation { - condition = alltrue([for s in var.subnetworks : can(s["subnet_ip"])]) || alltrue([for s in var.subnetworks : can(s["new_bits"])]) - error_message = "All subnetworks must make same choice of \"subnet_ip\" or \"new_bits\"." - } -} - -# tflint-ignore: terraform_unused_declarations -variable "primary_subnetwork" { - description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" - type = map(string) - default = null - validation { - condition = var.primary_subnetwork == null - error_message = "primary_subnetwork is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." - } -} - -# tflint-ignore: terraform_unused_declarations -variable "additional_subnetworks" { - description = "DEPRECATED: please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions" - type = list(map(string)) - default = null - validation { - condition = var.additional_subnetworks == null - error_message = "additional_subnetworks is deprecated. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." - } -} - -variable "secondary_ranges" { - type = map(list(object({ range_name = string, ip_cidr_range = string }))) - description = <<-EOT - "Secondary ranges associated with the subnets. - This will be deprecated in favour of secondary_ranges_list at a later date. - Please migrate to using the same." - EOT - default = {} -} - -variable "secondary_ranges_list" { - type = list(object({ - subnetwork_name = string, - ranges = list(object({ - range_name = string, - ip_cidr_range = string - })) - })) - description = <<-EOT - "List of secondary ranges associated with the subnetworks. - Each subnetwork must be specified at most once in this list." - EOT - default = [] - validation { - condition = (length(var.secondary_ranges_list[*].subnetwork_name) == - length(distinct(var.secondary_ranges_list[*].subnetwork_name))) - error_message = "Each subnetwork should be specified at most once in this list. Remove any duplicates." - } -} - -variable "network_routing_mode" { - type = string - default = "GLOBAL" - description = "The network routing mode (default \"GLOBAL\")" - - validation { - condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) - error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." - } -} - -variable "network_description" { - type = string - description = "An optional description of this resource (changes will trigger resource destroy/create)" - default = "" -} - -variable "ips_per_nat" { - type = number - description = "The number of IP addresses to allocate for each regional Cloud NAT (set to 0 to disable NAT)" - default = 2 -} - -variable "shared_vpc_host" { - type = bool - description = "Makes this project a Shared VPC host if 'true' (default 'false')" - default = false -} - -variable "delete_default_internet_gateway_routes" { - type = bool - description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" - default = false -} - -variable "enable_iap_ssh_ingress" { - type = bool - description = "Enable a firewall rule to allow SSH access using IAP tunnels" - default = true -} - -variable "enable_iap_rdp_ingress" { - type = bool - description = "Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels" - default = false -} - -variable "enable_iap_winrm_ingress" { - type = bool - description = "Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels" - default = false -} - -variable "enable_internal_traffic" { - type = bool - description = "Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network" - default = true -} - -variable "enable_cloud_router" { - type = bool - description = "Enable the creation of a Cloud Router for your VPC. For more information on Cloud Routers see https://cloud.google.com/network-connectivity/docs/router/concepts/overview" - default = true -} - -variable "enable_cloud_nat" { - type = bool - description = "Enable the creation of Cloud NATs." - default = true -} - -variable "extra_iap_ports" { - type = list(string) - description = "A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable_iap variables for standard ports)" - default = [] -} - -variable "allowed_ssh_ip_ranges" { - type = list(string) - description = "A list of CIDR IP ranges from which to allow ssh access" - default = [] - - validation { - condition = alltrue([for r in var.allowed_ssh_ip_ranges : can(cidrhost(r, 32))]) - error_message = "Each element of var.allowed_ssh_ip_ranges must be a valid CIDR-formatted IPv4 range." - } -} - -variable "firewall_rules" { - type = any - description = "List of firewall rules" - default = [] -} - -variable "firewall_log_config" { - type = string - description = "Firewall log configuration for Toolkit firewall rules (var.enable_iap_ssh_ingress and others)" - default = "DISABLE_LOGGING" - nullable = false - - validation { - condition = contains([ - "INCLUDE_ALL_METADATA", - "EXCLUDE_ALL_METADATA", - "DISABLE_LOGGING", - ], var.firewall_log_config) - error_message = "var.firewall_log_config must be set to \"DISABLE_LOGGING\", or enable logging with \"INCLUDE_ALL_METADATA\" or \"EXCLUDE_ALL_METADATA\"" - } -} - -variable "network_profile" { - type = string - description = <<-EOT - A full or partial URL of the network profile to apply to this network. - This field can be set only at resource creation time. For example, the - following are valid URLs: - - https://www.googleapis.com/compute/beta/projects/{projectId}/global/networkProfiles/{network_profile_name} - - projects/{projectId}/global/networkProfiles/{network_profile_name}} - When using a Mellanox network profile (contains 'roce'), if firewall_rules is specified or enable_internal_traffic is true, an error will be thrown - EOT - default = null -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/network/vpc/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/network/vpc/versions.tf deleted file mode 100644 index 71b7106734..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/network/vpc/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_version = ">= 0.15.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/README.md b/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/README.md deleted file mode 100644 index 192d7575a5..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/README.md +++ /dev/null @@ -1,320 +0,0 @@ -# Custom Images in the Cluster Toolkit (formerly HPC Toolkit) - -Please review the -[introduction to image building](../../../docs/image-building.md) for general -information on building custom images using the Toolkit. - -## Introduction - -This module uses [Packer](https://www.packer.io/) to create an image within an -Cluster Toolkit deployment. Packer operates by provisioning a short-lived VM in -Google Cloud on which it executes scripts to customize the boot disk for -repeated use. The VM's boot disk is specified from a source image that defaults -to the [HPC VM Image][hpcimage]. This Packer "template" supports customization -by the following approaches following a [recommended use](#recommended-use): - -- [startup-script metadata][startup-metadata] from [raw string][sss] or - [file][ssf] -- [Shell scripts][shell] uploaded from the Packer execution environment to the - VM -- [Ansible playbooks][ansible] uploaded from the Packer execution environment to - the VM - -They can be specified independently of one another, so that anywhere from 1 to 3 -solutions can be used simultaneously. In the case that 0 scripts are supplied, -the source boot disk is effectively copied to your project without -customization. This can be useful in scenarios where increased control over the -image maintenance lifecycle is desired or when policies restrict the use of -images to internal projects. - -## Minimum requirements - -### Outbound internet access - -Most customization scripts require access to resources on the public internet. -This can be achieved by one of the following 2 approaches: - -1. Using a public IP address on the VM - -- Set [var.omit_external_ip](#input_omit_external_ip) to `false` - -1. Configuring a VPC with a Cloud NAT in the region of the VM - -- Use the [vpc] module which automates NAT creation - -### Inbound internet access - -Read [order of execution](#order-of-execution) below for a discussion of VM -customization solutions and their requirements for inbound SSH access. -[Environments without SSH access](#environments-without-ssh-access) should use -the metadata-based startup-script solution. - -A simple way to enable inbound SSH access is to use the VPC module with -`allowed_ssh_ip_ranges` set to `0.0.0.0/0`. - -### User or service account executing Packer at command line - -The user or service account running Packer must have the permission to create -VMs in the selected VPC network and, if [use\_iap](#input_use_iap) is set, must -have the "IAP-Secured Tunnel User" role. Recommended roles are: - -- `roles/compute.instanceAdmin.v1` -- `roles/iap.tunnelResourceAccessor` - -### VM service account roles - -The service account attached to the temporary build VM created by Packer should -have the ability to write Cloud Logging entries so that you may inspect and -debug build logs. When using the metadata startup-script customization solution, -the service account attached to the temporary build VM created by Packer must -have the permission to modify its own metadata and to read from Cloud Storage -buckets. Recommended roles are: - -- `roles/compute.instanceAdmin.v1` -- `roles/iam.serviceAccountUser` -- `roles/logging.logWriter` -- `roles/monitoring.metricWriter` -- `roles/storage.objectViewer` - -It is recommended to create this service account as a separate step outside a -blueprint due to known delay in [IAM bindings propagation][iamprop]. - -## Example blueprints - -A recommended pattern for building images with this module is to use the -terraform based [startup-script] module along with this packer custom-image -module. Below you can find links to several examples of this pattern, including -usage instructions. - -### [Image Builder] - -The [Image Builder] blueprint demonstrates a solution that builds an image -using: - -- The [HPC VM Image][hpcimage] as a base upon which to customize -- A VPC network with firewall rules that allow IAP-based SSH tunnels -- A Toolkit runner that installs a custom script - -Please review the [examples README] for usage instructions. - -## Order of execution - -The startup script specified in metadata executes in parallel with the other -supported methods. However, the remaining methods execute in a well-defined -order relative to one another. - -1. All shell scripts will execute in the configured order -1. After shell scripts complete, all Ansible playbooks will execute in the - configured order - -> **_NOTE:_** if both [startup_script][sss] and [startup_script_file][ssf] are -> specified, then [startup_script_file][ssf] takes precedence. - -## Recommended use - -Because the [metadata startup script executes in parallel](#order-of-execution) -with the other solutions, conflicts can arise, especially when package managers -(`yum` or `apt`) lock their databases during package installation. Therefore, it -is recommended to choose one of the following approaches: - -1. Specify _either_ [startup_script][sss] _or_ [startup_script_file][ssf] and do - not specify [shell_scripts][shell] or [ansible_playbooks][ansible]. - - This can be especially useful in - [environments that restrict SSH access](#environments-without-ssh-access) -1. Specify any combination of [shell_scripts][shell] and - [ansible_playbooks][ansible] and do not specify [startup_script][sss] or - [startup_script_file][ssf]. - -If any of the startup script approaches fail by returning a code other than 0, -Packer will determine that the build has failed and refuse to save the image. - -## External access with SSH - -The [shell scripts][shell] and [Ansible playbooks][ansible] customization -solutions both require SSH access to the VM from the Packer execution -environment. SSH access can be enabled one of 2 ways: - -1. The VM is created without a public IP address and SSH tunnels are created - using [Identity-Aware Proxy (IAP)][iaptunnel]. - - Allow [use_iap](#input_use_iap) to take on its default value of `true` -1. The VM is created with an IP address on the public internet and firewall - rules allow SSH access from the Packer execution environment. - - Set `omit_external_ip = false` (or `omit_external_ip: false` in a - blueprint) - - Add firewall rules that open SSH to the VM - -The Packer template defaults to using to the 1st IAP-based solution because it -is more secure (no exposure to public internet) and because the [vpc] module -automatically sets up all necessary firewall rules for SSH tunneling and -outbound-only access to the internet through [Cloud NAT][cloudnat]. - -In either SSH solution, customization scripts should be supplied as files in the -[shell_scripts][shell] and [ansible_playbooks][ansible] settings. - -## Environments without SSH access - -Many network environments disallow SSH access to VMs. In these environments, the -[metadata-based startup scripts][startup-metadata] are appropriate because they -execute entirely independently of the Packer execution environment. - -In this scenario, a single scripts should be supplied in the form of a string to -the [startup_script][sss] input variable. This solution integrates well with -Toolkit runners. Runners operate by using a single startup script whose behavior -is extended by downloading and executing a customizable set of runners from -Cloud Storage at startup. - -> **_NOTE:_** Packer will attempt to use SSH if either [shell_scripts][shell] or -> [ansible_playbooks][ansible] are set to non-empty values. Leave them at their -> default, empty values to ensure access by SSH is disabled. - -## Supplying startup script as a string - -The [startup_script][sss] parameter accepts scripts formatted as strings. In -Packer and Terraform, multi-line strings can be specified using -[heredoc syntax](https://www.terraform.io/language/expressions/strings#heredoc-strings) -in an input [Packer variables file][pkrvars] (`*.pkrvars.hcl`) For example, the -following snippet defines a multi-line bash script followed by an integer -representing the size, in GiB, of the resulting image: - -```hcl -startup_script = <<-EOT - #!/bin/bash - yum install -y epel-release - yum install -y jq - EOT - -disk_size = 100 -``` - -In a blueprint, the equivalent syntax is: - -```yaml -... - settings: - startup_script: | - #!/bin/bash - yum install -y epel-release - yum install -y jq - disk_size: 100 -... -``` - -## Monitoring startup script execution - -When using startup script customization, Packer will print very limited output -to the console. For example: - -```text -==> example.googlecompute.toolkit_image: Waiting for any running startup script to finish... -==> example.googlecompute.toolkit_image: Startup script not finished yet. Waiting... -==> example.googlecompute.toolkit_image: Startup script not finished yet. Waiting... -==> example.googlecompute.toolkit_image: Startup script, if any, has finished running. -``` - -### Debugging startup-script failures - -> [!NOTE] -> There can be a delay in the propagation of the logs from the instance to -> Cloud Logging, so it may require waiting a few minutes to see the full logs. - -If the Packer image build fails, the module will output a `gcloud` command -that can be used directly to review startup-script execution. - -## License - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. You may obtain a copy of the -License at - -```text - http://www.apache.org/licenses/LICENSE-2.0 -``` - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. - - -## Requirements - -No requirements. - -## Providers - -No providers. - -## Modules - -No modules. - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [accelerator\_count](#input\_accelerator\_count) | Number of accelerator cards to attach to the VM; not necessary for families that always include GPUs (A2). | `number` | `null` | no | -| [accelerator\_type](#input\_accelerator\_type) | Type of accelerator cards to attach to the VM; not necessary for families that always include GPUs (A2). | `string` | `null` | no | -| [ansible\_playbooks](#input\_ansible\_playbooks) | A list of Ansible playbook configurations that will be uploaded to customize the VM image | 
list(object({
    playbook_file   = string
    galaxy_file     = string
    extra_arguments = list(string)
  }))
| `[]` | no | -| [communicator](#input\_communicator) | Communicator to use for provisioners that require access to VM ("ssh" or "winrm") | `string` | `null` | no | -| [deployment\_name](#input\_deployment\_name) | Cluster Toolkit deployment name | `string` | n/a | yes | -| [disk\_size](#input\_disk\_size) | Size of disk image in GB | `number` | `null` | no | -| [disk\_type](#input\_disk\_type) | Type of persistent disk to provision | `string` | `"pd-balanced"` | no | -| [enable\_shielded\_vm](#input\_enable\_shielded\_vm) | Enable the Shielded VM configuration (var.shielded\_instance\_config). | `bool` | `false` | no | -| [image\_family](#input\_image\_family) | The family name of the image to be built. Defaults to `deployment_name` | `string` | `null` | no | -| [image\_name](#input\_image\_name) | The name of the image to be built. If not supplied, it will be set to image\_family-$ISO\_TIMESTAMP | `string` | `null` | no | -| [image\_storage\_locations](#input\_image\_storage\_locations) | Storage location, either regional or multi-regional, where snapshot content is to be stored and only accepts 1 value.
See https://developer.hashicorp.com/packer/plugins/builders/googlecompute#image_storage_locations | `list(string)` | `null` | no | -| [labels](#input\_labels) | Labels to apply to the short-lived VM | `map(string)` | `null` | no | -| [machine\_type](#input\_machine\_type) | VM machine type on which to build new image | `string` | `"n2-standard-4"` | no | -| [manifest\_file](#input\_manifest\_file) | File to which to write Packer build manifest | `string` | `"packer-manifest.json"` | no | -| [metadata](#input\_metadata) | Instance metadata for the builder VM (use var.startup\_script or var.startup\_script\_file to set startup-script metadata) | `map(string)` | `{}` | no | -| [network\_project\_id](#input\_network\_project\_id) | Project ID of Shared VPC network | `string` | `null` | no | -| [omit\_external\_ip](#input\_omit\_external\_ip) | Provision the image building VM without a public IP address | `bool` | `true` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE` | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which to create VM and image | `string` | n/a | yes | -| [scopes](#input\_scopes) | DEPRECATED: use var.service\_account\_scopes | `set(string)` | `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | The service account email to use. If null or 'default', then the default Compute Engine service account will be used. | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Service account scopes to attach to the instance. See
https://cloud.google.com/compute/docs/access/service-accounts. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [shell\_scripts](#input\_shell\_scripts) | A list of paths to local shell scripts which will be uploaded to customize the VM image | `list(string)` | `[]` | no | -| [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance (must set var.enabled\_shielded\_vm) | 
object({
    enable_secure_boot          = bool
    enable_vtpm                 = bool
    enable_integrity_monitoring = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | -| [source\_image](#input\_source\_image) | Source OS image to build from | `string` | `null` | no | -| [source\_image\_family](#input\_source\_image\_family) | Alternative to source\_image. Specify image family to build from latest image in family | `string` | `"hpc-rocky-linux-8"` | no | -| [source\_image\_project\_id](#input\_source\_image\_project\_id) | A list of project IDs to search for the source image. Packer will search the
first project ID in the list first, and fall back to the next in the list,
until it finds the source image. | `list(string)` | `null` | no | -| [ssh\_username](#input\_ssh\_username) | Username to use for SSH access to VM | `string` | `"hpc-toolkit-packer"` | no | -| [startup\_script](#input\_startup\_script) | Startup script (as raw string) used to build the custom Linux VM image (overridden by var.startup\_script\_file if both are set) | `string` | `null` | no | -| [startup\_script\_file](#input\_startup\_script\_file) | File path to local shell script that will be used to customize the Linux VM image (overrides var.startup\_script) | `string` | `null` | no | -| [state\_timeout](#input\_state\_timeout) | The time to wait for instance state changes, including image creation | `string` | `"10m"` | no | -| [subnetwork\_name](#input\_subnetwork\_name) | Name of subnetwork in which to provision image building VM | `string` | n/a | yes | -| [tags](#input\_tags) | Assign network tags to apply firewall rules to VM instance | `list(string)` | `null` | no | -| [use\_iap](#input\_use\_iap) | Use IAP proxy when connecting by SSH | `bool` | `true` | no | -| [use\_os\_login](#input\_use\_os\_login) | Use OS Login when connecting by SSH | `bool` | `false` | no | -| [windows\_startup\_ps1](#input\_windows\_startup\_ps1) | A list of strings containing PowerShell scripts which will customize a Windows VM image (requires WinRM communicator) | `list(string)` | `[]` | no | -| [wrap\_startup\_script](#input\_wrap\_startup\_script) | Wrap startup script with Packer-generated wrapper | `bool` | `true` | no | -| [zone](#input\_zone) | Cloud zone in which to provision image building VM | `string` | n/a | yes | - -## Outputs - -No outputs. - - -[ansible]: #input_ansible_playbooks -[cloudnat]: https://cloud.google.com/nat/docs/overview -[examples readme]: ../../../examples/README.md#image-builderyaml- -[hpcimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm -[iamprop]: https://cloud.google.com/iam/docs/access-change-propagation -[iaptunnel]: https://cloud.google.com/iap/docs/using-tcp-forwarding -[image builder]: ../../../examples/image-builder.yaml -[logging-console]: https://console.cloud.google.com/logs/ -[logging-read-docs]: https://cloud.google.com/sdk/gcloud/reference/logging/read -[pkrvars]: https://www.packer.io/guides/hcl/variables#from-a-file -[shell]: #input_shell_scripts -[ssf]: #input_startup_script_file -[sss]: #input_startup_script -[startup-metadata]: https://cloud.google.com/compute/docs/instances/startup-scripts/linux -[startup-script]: ../../../modules/scripts/startup-script -[vpc]: ../../network/vpc/README.md diff --git a/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/image.pkr.hcl b/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/image.pkr.hcl deleted file mode 100644 index 9282cf7433..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/image.pkr.hcl +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "custom-image", ghpc_role = "packer" }) - - # construct a unique image name from the image family - image_family = var.image_family != null ? var.image_family : var.deployment_name - image_name_default = "${local.image_family}-${formatdate("YYYYMMDD't'hhmmss'z'", timestamp())}" - image_name = var.image_name != null ? var.image_name : local.image_name_default - - # construct vm image name for use when getting logs - instance_name = "packer-${substr(uuidv4(), 0, 6)}" - - # default to explicit var.communicator, otherwise in-order: ssh/winrm/none - shell_script_communicator = length(var.shell_scripts) > 0 ? "ssh" : "" - ansible_playbook_communicator = length(var.ansible_playbooks) > 0 ? "ssh" : "" - powershell_script_communicator = length(var.windows_startup_ps1) > 0 ? "winrm" : "" - communicator = coalesce( - var.communicator, - local.shell_script_communicator, - local.ansible_playbook_communicator, - local.powershell_script_communicator, - "none" - ) - - # must not enable IAP when no communicator is in use - use_iap = local.communicator == "none" ? false : var.use_iap - - # construct metadata from startup_script and metadata variables - startup_script_metadata = var.startup_script == null ? {} : { startup-script = var.startup_script } - - linux_user_metadata = { - block-project-ssh-keys = "TRUE" - shutdown-script = <<-EOT - #!/bin/bash - userdel -r ${var.ssh_username} - sed -i '/${var.ssh_username}/d' /var/lib/google/google_users - EOT - } - windows_packer_user = "packer_user" - windows_user_metadata = { - sysprep-specialize-script-cmd = "winrm quickconfig -quiet & net user /add ${local.windows_packer_user} & net localgroup administrators ${local.windows_packer_user} /add & winrm set winrm/config/service/auth @{Basic=\\\"true\\\"}" - windows-shutdown-script-cmd = <<-EOT - net user /delete ${local.windows_packer_user} - EOT - } - user_metadata = local.communicator == "winrm" ? local.windows_user_metadata : local.linux_user_metadata - - # merge metadata such that var.metadata always overrides user management - # metadata but always allow var.startup_script to override var.metadata - metadata = merge( - local.user_metadata, - var.metadata, - local.startup_script_metadata, - ) - - # determine best value for on_host_maintenance if not supplied by user - machine_vals = split("-", var.machine_type) - machine_family = local.machine_vals[0] - gpu_attached = contains(["a2", "g2"], local.machine_family) || var.accelerator_type != null - on_host_maintenance_default = local.gpu_attached ? "TERMINATE" : "MIGRATE" - on_host_maintenance = ( - var.on_host_maintenance != null - ? var.on_host_maintenance - : local.on_host_maintenance_default - ) - - accelerator_type = var.accelerator_type == null ? null : "projects/${var.project_id}/zones/${var.zone}/acceleratorTypes/${var.accelerator_type}" - - winrm_username = local.communicator == "winrm" ? "packer_user" : null - winrm_insecure = local.communicator == "winrm" ? true : null - winrm_use_ssl = local.communicator == "winrm" ? true : null - - enable_integrity_monitoring = var.enable_shielded_vm && var.shielded_instance_config.enable_integrity_monitoring - enable_secure_boot = var.enable_shielded_vm && var.shielded_instance_config.enable_secure_boot - enable_vtpm = var.enable_shielded_vm && var.shielded_instance_config.enable_vtpm - - image_licenses = [ - "projects/click-to-deploy-images/global/licenses/hpc-toolkit-vm-image" - ] -} - -source "googlecompute" "toolkit_image" { - communicator = local.communicator - project_id = var.project_id - image_name = local.image_name - image_family = local.image_family - image_labels = local.labels - instance_name = local.instance_name - machine_type = var.machine_type - accelerator_type = local.accelerator_type - accelerator_count = var.accelerator_count - on_host_maintenance = local.on_host_maintenance - disk_size = var.disk_size - disk_type = var.disk_type - omit_external_ip = var.omit_external_ip - use_internal_ip = var.omit_external_ip - subnetwork = var.subnetwork_name - network_project_id = var.network_project_id - service_account_email = var.service_account_email - scopes = var.service_account_scopes - source_image = var.source_image - source_image_family = var.source_image_family - source_image_project_id = var.source_image_project_id - ssh_username = var.ssh_username - tags = var.tags - use_iap = local.use_iap - use_os_login = var.use_os_login - winrm_username = local.winrm_username - winrm_insecure = local.winrm_insecure - winrm_use_ssl = local.winrm_use_ssl - zone = var.zone - labels = local.labels - metadata = local.metadata - startup_script_file = var.startup_script_file - wrap_startup_script = var.wrap_startup_script - state_timeout = var.state_timeout - image_storage_locations = var.image_storage_locations - enable_secure_boot = local.enable_secure_boot - enable_vtpm = local.enable_vtpm - enable_integrity_monitoring = local.enable_integrity_monitoring - image_licenses = local.image_licenses -} - -build { - name = var.deployment_name - sources = ["sources.googlecompute.toolkit_image"] - - # using dynamic blocks to create provisioners ensures that there are no - # provisioner blocks when none are provided and we can use the none - # communicator when using startup-script - - # provisioner "shell" blocks - dynamic "provisioner" { - labels = ["shell"] - for_each = var.shell_scripts - content { - execute_command = "sudo -H sh -c '{{ .Vars }} {{ .Path }}'" - script = provisioner.value - } - } - - # provisioner "powershell" blocks - dynamic "provisioner" { - labels = ["powershell"] - for_each = var.windows_startup_ps1 - content { - inline = split("\n", provisioner.value) - } - } - - dynamic "provisioner" { - labels = ["powershell"] - for_each = length(var.windows_startup_ps1) > 0 ? [1] : [] - content { - inline = [ - "GCESysprep -no_shutdown" - ] - } - } - - # provisioner "ansible-local" blocks - # this installs custom roles/collections from ansible-galaxy in /home/packer - # which will be removed at the end; consider modifying /etc/ansible/ansible.cfg - dynamic "provisioner" { - labels = ["ansible-local"] - for_each = var.ansible_playbooks - content { - playbook_file = provisioner.value.playbook_file - galaxy_file = provisioner.value.galaxy_file - extra_arguments = provisioner.value.extra_arguments - } - } - - post-processor "manifest" { - output = var.manifest_file - strip_path = true - custom_data = { - built-by = "cloud-hpc-toolkit" - } - } - - # If there is an error during image creation, print out command for getting packer VM logs - error-cleanup-provisioner "shell-local" { - environment_vars = [ - "PRJ_ID=${var.project_id}", - "INST_NAME=${local.instance_name}", - "ZONE=${var.zone}", - ] - inline_shebang = "/bin/bash -e" - inline = [ - "type -P gcloud > /dev/null || exit 0", - "INST_ID=$(gcloud compute instances describe $INST_NAME --project $PRJ_ID --format=\"value(id)\" --zone=$ZONE)", - "echo 'Error building image try checking logs:'", - join(" ", ["echo \"gcloud logging --project $PRJ_ID read", - "'logName=(\\\"projects/$PRJ_ID/logs/GCEMetadataScripts\\\" OR \\\"projects/$PRJ_ID/logs/google_metadata_script_runner\\\") AND resource.labels.instance_id=$INST_ID'", - "--format=\\\"table(timestamp, resource.labels.instance_id, jsonPayload.message)\\\"", - "--order=asc\"" - ] - ) - ] - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/metadata.yaml deleted file mode 100644 index 23108c4e17..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - compute.googleapis.com - - logging.googleapis.com - - storage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/variables.pkr.hcl b/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/variables.pkr.hcl deleted file mode 100644 index 3cede102ce..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/packer/custom-image/variables.pkr.hcl +++ /dev/null @@ -1,276 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "deployment_name" { - description = "Cluster Toolkit deployment name" - type = string -} - -variable "project_id" { - description = "Project in which to create VM and image" - type = string -} - -variable "machine_type" { - description = "VM machine type on which to build new image" - type = string - default = "n2-standard-4" -} - -variable "disk_size" { - description = "Size of disk image in GB" - type = number - default = null -} - -variable "disk_type" { - description = "Type of persistent disk to provision" - type = string - default = "pd-balanced" -} - -variable "zone" { - description = "Cloud zone in which to provision image building VM" - type = string -} - -variable "network_project_id" { - description = "Project ID of Shared VPC network" - type = string - default = null -} - -variable "subnetwork_name" { - description = "Name of subnetwork in which to provision image building VM" - type = string -} - -variable "omit_external_ip" { - description = "Provision the image building VM without a public IP address" - type = bool - default = true -} - -variable "tags" { - description = "Assign network tags to apply firewall rules to VM instance" - type = list(string) - default = null -} - -variable "image_family" { - description = "The family name of the image to be built. Defaults to `deployment_name`" - type = string - default = null -} - -variable "image_name" { - description = "The name of the image to be built. If not supplied, it will be set to image_family-$ISO_TIMESTAMP" - type = string - default = null -} - -variable "source_image_project_id" { - description = < -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.1 | -| [google](#requirement\_google) | >= 4.0 | -| [local](#requirement\_local) | >= 2.0.0 | -| [null](#requirement\_null) | ~> 3.0 | -| [random](#requirement\_random) | >= 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 4.0 | -| [local](#provider\_local) | >= 2.0.0 | -| [null](#provider\_null) | ~> 3.0 | -| [random](#provider\_random) | >= 3.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [instance\_template](#module\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 12.1 | -| [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | ../../scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [local_file.job_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [local_file.submit_script](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [null_resource.submit_job](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [random_id.submit_job_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | -| [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [allow\_automatic\_updates](#input\_allow\_automatic\_updates) | If false, disables automatic system package updates on the created instances. This feature is
only available on supported images (or images derived from them). For more details, see
https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates | `bool` | `true` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment, used for the job\_id | `string` | n/a | yes | -| [enable\_public\_ips](#input\_enable\_public\_ips) | If set to true, instances will have public IPs | `bool` | `true` | no | -| [gcloud\_version](#input\_gcloud\_version) | The version of the gcloud cli being used. Used for output instructions. Valid inputs are `"alpha"`, `"beta"` and "" (empty string for default version) | `string` | `""` | no | -| [image](#input\_image) | DEPRECATED: Google Cloud Batch compute node image. Ignored if `instance_template` is provided. | `any` | `null` | no | -| [instance\_image](#input\_instance\_image) | Google Cloud Batch compute node image. Ignored if `instance_template` is provided.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "hpc-rocky-linux-8",
  "project": "cloud-hpc-image-public"
}
| no | -| [instance\_template](#input\_instance\_template) | Compute VM instance template self-link to be used for Google Cloud Batch compute node. If provided, a number of other variables will be ignored as noted by `Ignored if instance_template is provided` in descriptions. | `string` | `null` | no | -| [job\_filename](#input\_job\_filename) | The filename of the generated job template file. Will default to `cloud-batch-.json` if not specified | `string` | `null` | no | -| [job\_id](#input\_job\_id) | An id for the Google Cloud Batch job. Used for output instructions and file naming. Automatically populated by the module id if not set. If setting manually, ensure a unique value across all jobs. | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the Google Cloud Batch compute nodes. Key-value pairs. Ignored if `instance_template` is provided. | `map(string)` | n/a | yes | -| [log\_policy](#input\_log\_policy) | Create a block to define log policy.
When set to `CLOUD_LOGGING`, logs will be sent to Cloud Logging.
When set to `PATH`, path must be added to generated template.
When set to `DESTINATION_UNSPECIFIED`, logs will not be preserved. | `string` | `"CLOUD_LOGGING"` | no | -| [machine\_type](#input\_machine\_type) | Machine type to use for Google Cloud Batch compute nodes. Ignored if `instance_template` is provided. | `string` | `"n2-standard-4"` | no | -| [mpi\_mode](#input\_mpi\_mode) | Sets up barriers before and after each runnable. In addition, sets `permissiveSsh=true`, `requireHostsFile=true`, and `taskCountPerNode=1`. `taskCountPerNode` can be overridden by `task_count_per_node`. | `bool` | `false` | no | -| [native\_batch\_mounting](#input\_native\_batch\_mounting) | Batch can mount some fs\_type nativly using the 'volumes' block in the job file. If set to false, all mounting will happen through Cluster Toolkit startup scripts. | `bool` | `true` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. Ignored if `instance_template` is provided. | 
list(object({
    server_ip             = string
    remote_mount          = string
    local_mount           = string
    fs_type               = string
    mount_options         = string
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE` | `string` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The region in which to run the Google Cloud Batch job | `string` | n/a | yes | -| [runnable](#input\_runnable) | A simplified form of `var.runnables` that only takes a single script. Use either `runnables` or `runnable`. | `string` | `null` | no | -| [runnables](#input\_runnables) | A list of shell scripts to be executed in sequence as the main workload of the Google Batch job. These will be used to populate the generated template. | 
list(object({
    script = string
  }))
| `null` | no | -| [service\_account](#input\_service\_account) | Service account to attach to the Google Cloud Batch compute node. Ignored if `instance_template` is provided. | 
object({
    email  = string,
    scopes = set(string)
  })
| 
{
  "email": null,
  "scopes": [
    "https://www.googleapis.com/auth/devstorage.read_only",
    "https://www.googleapis.com/auth/logging.write",
    "https://www.googleapis.com/auth/monitoring.write",
    "https://www.googleapis.com/auth/servicecontrol",
    "https://www.googleapis.com/auth/service.management.readonly",
    "https://www.googleapis.com/auth/trace.append"
  ]
}
| no | -| [startup\_script](#input\_startup\_script) | Startup script run before Google Cloud Batch job starts. Ignored if `instance_template` is provided. | `string` | `null` | no | -| [submit](#input\_submit) | When set to true, the generated job file will be submitted automatically to Google Cloud as part of terraform apply. | `bool` | `false` | no | -| [subnetwork](#input\_subnetwork) | The subnetwork that the Batch job should run on. Defaults to 'default' subnet. Ignored if `instance_template` is provided. | `any` | `null` | no | -| [task\_count](#input\_task\_count) | Number of parallel tasks | `number` | `1` | no | -| [task\_count\_per\_node](#input\_task\_count\_per\_node) | Max number of tasks that can be run on a VM at the same time. If not specified, Batch will decide a value. | `number` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [gcloud\_version](#output\_gcloud\_version) | The version of gcloud to be used. | -| [instance\_template](#output\_instance\_template) | Instance template used by the Batch job. | -| [instructions](#output\_instructions) | Instructions for submitting the Batch job. | -| [job\_data](#output\_job\_data) | All data associated with the defined job, typically provided as input to clout-batch-login-node. | -| [network\_storage](#output\_network\_storage) | An array of network attached storage mounts used by the Batch job. | -| [startup\_script](#output\_startup\_script) | Startup script run before Google Cloud Batch job starts. | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf deleted file mode 100644 index 7a7fe02307..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/compute_image.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_compute_image" "compute_image" { - family = try(var.instance_image.family, null) - name = try(var.instance_image.name, null) - project = try(var.instance_image.project, null) - - lifecycle { - postcondition { - # Condition needs to check the suffix of the license, as prefix contains an API version which can change. - # Example license value: https://www.googleapis.com/compute/v1/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates - condition = var.allow_automatic_updates || anytrue([for license in self.licenses : endswith(license, "/projects/cloud-hpc-image-public/global/licenses/hpc-vm-image-feature-disable-auto-updates")]) - error_message = "Disabling automatic updates is not supported with the selected VM image. More information: https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates" - } - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/main.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/main.tf deleted file mode 100644 index 0d681536c9..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/main.tf +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "batch-job-template", ghpc_role = "scheduler" }) -} - -locals { - instance_template = coalesce(var.instance_template, module.instance_template.self_link) - - tasks_per_node = var.task_count_per_node != null ? var.task_count_per_node : (var.mpi_mode ? 1 : null) - - one_line_runnable = coalesce(var.runnable, "## Add your workload here ##") - runnables = coalesce(var.runnables, [{ script = local.one_line_runnable }]) - - job_template_contents = templatefile( - "${path.module}/templates/batch-job-base.yaml.tftpl", - { - synchronized = var.mpi_mode - runnables = local.runnables - task_count = var.task_count - tasks_per_node = local.tasks_per_node - require_hosts_file = var.mpi_mode - permissive_ssh = var.mpi_mode - log_policy = var.log_policy - instance_template = local.instance_template - nfs_volumes = local.native_batch_network_storage - labels = local.labels - } - ) - - submit_job_id = "${var.job_id}-${random_id.submit_job_suffix.hex}" - job_filename = coalesce(var.job_filename, "${var.job_id}.yaml") - job_template_output_path = "${path.root}/${local.job_filename}" - - submit_script_contents = templatefile( - "${path.module}/templates/batch-submit.sh.tftpl", - { - project = var.project_id - location = var.region - config = local_file.job_template.filename - submit_job_id = local.submit_job_id - } - ) - submit_script_output_path = "${path.root}/submit-${var.job_id}.sh" - - subnetwork_name = var.subnetwork != null ? var.subnetwork.name : "default" - subnetwork_project = var.subnetwork != null ? var.subnetwork.project : var.project_id - - # Filter network_storage for native Batch support - native_fstype = var.native_batch_mounting ? ["nfs"] : [] - native_batch_network_storage = [ - for ns in var.network_storage : - ns if contains(local.native_fstype, ns.fs_type) - ] - # other processing happens in startup_from_network_storage.tf - - # this code is similar to code in Packer and vm-instance modules - # it differs in that this module does not (yet) expose var.guest_acclerator - # for attaching GPUs to N1 VMs. For now, identify only A2 types. - machine_vals = split("-", var.machine_type) - machine_family = local.machine_vals[0] - gpu_attached = contains(["a2", "g2"], local.machine_family) - on_host_maintenance_default = local.gpu_attached ? "TERMINATE" : "MIGRATE" - - on_host_maintenance = coalesce(var.on_host_maintenance, local.on_host_maintenance_default) - - network_storage_metadata = var.network_storage != null ? ({ network_storage = jsonencode(var.network_storage) }) : {} - disable_automatic_updates_metadata = var.allow_automatic_updates ? {} : { google_disable_automatic_updates = "TRUE" } - - metadata = merge( - local.network_storage_metadata, - local.disable_automatic_updates_metadata - ) -} - -module "instance_template" { - source = "terraform-google-modules/vm/google//modules/instance_template" - version = "~> 12.1" - - name_prefix = var.instance_template == null ? "${var.job_id}-instance-template" : "unused-template" - project_id = var.project_id - subnetwork = local.subnetwork_name - subnetwork_project = local.subnetwork_project - service_account = var.service_account - access_config = var.enable_public_ips ? [{ nat_ip = null, network_tier = null }] : [] - labels = local.labels - - machine_type = var.machine_type - startup_script = local.startup_from_network_storage - metadata = local.metadata - source_image_family = data.google_compute_image.compute_image.family - source_image = data.google_compute_image.compute_image.name - source_image_project = data.google_compute_image.compute_image.project - on_host_maintenance = local.on_host_maintenance -} - -resource "local_file" "job_template" { - content = local.job_template_contents - filename = local.job_template_output_path - - lifecycle { - precondition { - condition = var.runnable == null || var.runnables == null - error_message = "var.runnable and var.runnables (plural) cannot both be set." - } - } -} - -resource "random_id" "submit_job_suffix" { - byte_length = 4 - keepers = { - always_run = timestamp() - } -} - -resource "local_file" "submit_script" { - content = local.submit_script_contents - filename = local.submit_script_output_path -} - -resource "null_resource" "submit_job" { - depends_on = [local_file.job_template, local_file.submit_script] - count = var.submit ? 1 : 0 - - # A new deployment should always submit a new job. Old finished jobs aren't persistent parts of - # Cloud infrastructure. - triggers = { - always_run = timestamp() - } - - provisioner "local-exec" { - command = local.submit_script_output_path - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml deleted file mode 100644 index 387e810962..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/metadata.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - batch.googleapis.com - - compute.googleapis.com -ghpc: - inject_module_id: job_id diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/outputs.tf deleted file mode 100644 index 0b1295975a..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/outputs.tf +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - provided_instance_tpl_msg = "The Batch job template uses the existing VM instance template:" - generated_instance_tpl_msg = "The Batch job template uses a new VM instance template created matching the provided settings:" - submit_msg = <<-EOT - - The job has been submitted. See job status at: - https://console.cloud.google.com/batch/jobsDetail/regions/${var.region}/jobs/${local.submit_job_id}?project=${var.project_id} - EOT -} - -output "instructions" { - description = "Instructions for submitting the Batch job." - value = <<-EOT - - A Batch job template file has been created locally at: - ${abspath(local.job_template_output_path)} - - ${var.instance_template == null ? local.generated_instance_tpl_msg : local.provided_instance_tpl_msg} - ${local.instance_template} - ${var.submit ? local.submit_msg : ""} - - Use the following commands to: - Submit your job${var.submit ? " (Note: job has already been submitted)" : ""}: - gcloud ${var.gcloud_version} batch jobs submit ${local.submit_job_id} --config=${abspath(local.job_template_output_path)} --location=${var.region} --project=${var.project_id} - - Check status: - gcloud ${var.gcloud_version} batch jobs describe ${local.submit_job_id} --location=${var.region} --project=${var.project_id} | grep state: - - Delete job: - gcloud ${var.gcloud_version} batch jobs delete ${local.submit_job_id} --location=${var.region} --project=${var.project_id} - - List all jobs: - gcloud ${var.gcloud_version} batch jobs list --project=${var.project_id} - EOT -} - -output "job_data" { - description = "All data associated with the defined job, typically provided as input to clout-batch-login-node." - value = { - template_contents = local.job_template_contents, - filename = local.job_filename, - id = local.submit_job_id - } -} - -output "instance_template" { - description = "Instance template used by the Batch job." - value = local.instance_template -} - -output "network_storage" { - description = "An array of network attached storage mounts used by the Batch job." - value = var.network_storage -} - -output "startup_script" { - description = "Startup script run before Google Cloud Batch job starts." - value = var.startup_script -} - -output "gcloud_version" { - description = "The version of gcloud to be used." - value = var.gcloud_version -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf deleted file mode 100644 index 02bc58e4f7..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/startup_from_network_storage.tf +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -# This file is meant to be reused by multiple modules. -# "inputs": -# local.native_fstype : list of file systems that are supported automatically, but looking at the metadata. -# var.network_storage : to be passed into metadata somewhere else (not here) -# var.startup_script : to be changed into a more complete file system with all the fs runners - -# "outputs": -# local.startup_from_network_storage : A full startup script with all the runners that are not supported -# natively and were included in the network_storage structure - -locals { - startup_script_network_storage = [ - for ns in var.network_storage : - ns if !contains(local.native_fstype, ns.fs_type) - ] - # Pull out runners to include in startup script - storage_client_install_runners = [ - for ns in local.startup_script_network_storage : - ns.client_install_runner if ns.client_install_runner != null - ] - mount_runners = [ - for ns in local.startup_script_network_storage : - ns.mount_runner if ns.mount_runner != null - ] - - startup_script_runner = [{ - content = var.startup_script != null ? var.startup_script : "echo 'No user provided startup script.'" - destination = "passed_startup_script.sh" - type = "shell" - }] - - full_runner_list = concat( - local.storage_client_install_runners, - local.mount_runners, - local.startup_script_runner - ) - - startup_from_network_storage = module.netstorage_startup_script.startup_script -} - -module "netstorage_startup_script" { - source = "../../scripts/startup-script" - - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = local.full_runner_list -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl deleted file mode 100644 index 83fccde53b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-job-base.yaml.tftpl +++ /dev/null @@ -1,53 +0,0 @@ -taskGroups: - - taskSpec: - runnables: - %{~ if synchronized ~} - - barrier: - name: "wait-for-node-startup" - %{~ endif ~} - %{~ for runnable in runnables ~} - - script: - text: ${indent(12, chomp(yamlencode(runnable.script)))} - %{~ if synchronized ~} - - barrier: - name: "wait-for-script-to-complete" - %{~ endif ~} - %{~ endfor ~} - %{~ if length(nfs_volumes) > 0 ~} - volumes: - %{~ for index, vol in nfs_volumes ~} - - nfs: - server: "${vol.server_ip}" - remotePath: "${vol.remote_mount}" - %{~ if vol.mount_options != "" && vol.mount_options != null ~} - mountOptions: "${vol.mount_options}" - %{~ endif ~} - mountPath: "${vol.local_mount}" - %{~ endfor ~} - %{~ endif ~} - taskCount: ${task_count} - %{~ if tasks_per_node != null ~} - taskCountPerNode: ${tasks_per_node} - %{~ endif ~} - requireHostsFile: ${require_hosts_file} - permissiveSsh: ${permissive_ssh} -%{~ if instance_template != null } -allocationPolicy: - instances: - - instanceTemplate: "${instance_template}" -%{~ endif } -%{~ if log_policy == "CLOUD_LOGGING" } -logsPolicy: - destination: "CLOUD_LOGGING" -%{ endif } -%{~ if log_policy == "PATH" } -logsPolicy: - destination: "PATH" - logsPath: ## Add logging path here -%{ endif } -%{~ if length(labels) > 0 ~} -labels: -%{ for k, v in labels ~} - ${k}: "${v}" -%{ endfor } -%{~ endif ~} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl deleted file mode 100644 index 25f89c3ceb..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/templates/batch-submit.sh.tftpl +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -e -o pipefail -GCLOUD_MAJOR_VERSION=$(gcloud --version | head -n 1 | awk '{print $NF}' | cut -f1 --delimiter=.) -if [ $((GCLOUD_MAJOR_VERSION >= 461)) ]; then - gcloud batch jobs submit ${submit_job_id} --project=${project} --location=${location} --config=${config} - echo "batch job ${submit_job_id} successfully submitted" -else - echo "gcloud must be updated to version 461.0.0 or later." - exit 1 -fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/variables.tf deleted file mode 100644 index f65fbd111e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/variables.tf +++ /dev/null @@ -1,240 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "region" { - description = "The region in which to run the Google Cloud Batch job" - type = string -} - -variable "deployment_name" { - description = "Name of the deployment, used for the job_id" - type = string -} - -variable "labels" { - description = "Labels to add to the Google Cloud Batch compute nodes. Key-value pairs. Ignored if `instance_template` is provided." - type = map(string) -} - -variable "job_id" { - description = "An id for the Google Cloud Batch job. Used for output instructions and file naming. Automatically populated by the module id if not set. If setting manually, ensure a unique value across all jobs." - type = string -} - -variable "job_filename" { - description = "The filename of the generated job template file. Will default to `cloud-batch-.json` if not specified" - type = string - default = null -} - -variable "gcloud_version" { - description = "The version of the gcloud cli being used. Used for output instructions. Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default version)" - type = string - default = "" - - validation { - condition = contains(["alpha", "beta", ""], var.gcloud_version) - error_message = "Allowed values for gcloud_version are 'alpha', 'beta', or '' (empty string)." - } -} - -variable "task_count" { - description = "Number of parallel tasks" - type = number - default = 1 -} - -variable "task_count_per_node" { - description = "Max number of tasks that can be run on a VM at the same time. If not specified, Batch will decide a value." - type = number - default = null -} - -variable "mpi_mode" { - description = "Sets up barriers before and after each runnable. In addition, sets `permissiveSsh=true`, `requireHostsFile=true`, and `taskCountPerNode=1`. `taskCountPerNode` can be overridden by `task_count_per_node`." - type = bool - default = false -} - -variable "log_policy" { - description = <<-EOT - Create a block to define log policy. - When set to `CLOUD_LOGGING`, logs will be sent to Cloud Logging. - When set to `PATH`, path must be added to generated template. - When set to `DESTINATION_UNSPECIFIED`, logs will not be preserved. - EOT - type = string - default = "CLOUD_LOGGING" - - validation { - condition = contains(["CLOUD_LOGGING", "PATH", "DESTINATION_UNSPECIFIED"], var.log_policy) - error_message = "Allowed values for log_policy are 'CLOUD_LOGGING', 'PATH', or 'DESTINATION_UNSPECIFIED'." - } -} - -variable "runnables" { - description = "A list of shell scripts to be executed in sequence as the main workload of the Google Batch job. These will be used to populate the generated template." - type = list(object({ - script = string - })) - default = null -} - -variable "runnable" { - description = "A simplified form of `var.runnables` that only takes a single script. Use either `runnables` or `runnable`." - type = string - default = null -} - -variable "instance_template" { - description = "Compute VM instance template self-link to be used for Google Cloud Batch compute node. If provided, a number of other variables will be ignored as noted by `Ignored if instance_template is provided` in descriptions." - type = string - default = null -} - -variable "subnetwork" { - description = "The subnetwork that the Batch job should run on. Defaults to 'default' subnet. Ignored if `instance_template` is provided." - type = any - default = null -} - -variable "enable_public_ips" { - description = "If set to true, instances will have public IPs" - type = bool - default = true -} - -variable "service_account" { - description = "Service account to attach to the Google Cloud Batch compute node. Ignored if `instance_template` is provided." - type = object({ - email = string, - scopes = set(string) - }) - default = { - email = null - scopes = [ - "https://www.googleapis.com/auth/devstorage.read_only", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring.write", - "https://www.googleapis.com/auth/servicecontrol", - "https://www.googleapis.com/auth/service.management.readonly", - "https://www.googleapis.com/auth/trace.append" - ] - } -} - -variable "machine_type" { - description = "Machine type to use for Google Cloud Batch compute nodes. Ignored if `instance_template` is provided." - type = string - default = "n2-standard-4" -} - -variable "startup_script" { - description = "Startup script run before Google Cloud Batch job starts. Ignored if `instance_template` is provided." - type = string - default = null -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured. Ignored if `instance_template` is provided." - type = list(object({ - server_ip = string - remote_mount = string - local_mount = string - fs_type = string - mount_options = string - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "native_batch_mounting" { - description = "Batch can mount some fs_type nativly using the 'volumes' block in the job file. If set to false, all mounting will happen through Cluster Toolkit startup scripts." - type = bool - default = true -} - -# Deprecated, replaced by instance_image -# tflint-ignore: terraform_unused_declarations -variable "image" { - description = "DEPRECATED: Google Cloud Batch compute node image. Ignored if `instance_template` is provided." - type = any - default = null - - validation { - condition = var.image == null - error_message = "The 'var.image' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." - } -} - -variable "instance_image" { - description = <<-EOD - Google Cloud Batch compute node image. Ignored if `instance_template` is provided. - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "cloud-hpc-image-public" - family = "hpc-rocky-linux-8" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} - -variable "on_host_maintenance" { - description = "Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except the use of GPUs requires it to be `TERMINATE`" - type = string - default = null - validation { - condition = var.on_host_maintenance == null ? true : contains(["MIGRATE", "TERMINATE"], var.on_host_maintenance) - error_message = "When set, the on_host_maintenance must be set to MIGRATE or TERMINATE." - } -} - -variable "submit" { - description = "When set to true, the generated job file will be submitted automatically to Google Cloud as part of terraform apply." - type = bool - default = false -} - -variable "allow_automatic_updates" { - description = <<-EOT - If false, disables automatic system package updates on the created instances. This feature is - only available on supported images (or images derived from them). For more details, see - https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates - EOT - type = bool - default = true - nullable = false -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/versions.tf deleted file mode 100644 index a1161e1354..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-job-template/versions.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - null = { - source = "hashicorp/null" - version = "~> 3.0" - } - local = { - source = "hashicorp/local" - version = ">= 2.0.0" - } - random = { - source = "hashicorp/random" - version = ">= 3.0" - } - google = { - source = "hashicorp/google" - version = ">= 4.0" - } - } - required_version = ">= 1.1" -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/README.md b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/README.md deleted file mode 100644 index c20ca7dbeb..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/README.md +++ /dev/null @@ -1,127 +0,0 @@ -# Description - -This module creates a VM that acts as a login node to test and submit Google -Cloud Batch jobs. It is intended to be used along with the `batch-job-template` -module. - -This login node: - -- Uses the same VM settings as the first provided `batch-job-template`, such as - image, machine type, etc... -- Runs the same `startup-script` as the first provided `batch-job-template`. -- Has the same mounted file systems as the provided `batch-job-template`. -- Contains a folder with job templates generated by `batch-job-template` modules. - -Since the login node has the same mounted storage and is a homogeneous machine -to the Google Cloud Batch compute VMs, it can be used to inspect shared file -systems and test installed software before submitting a Google Cloud Batch job. - -## Example - -```yaml -- id: batch-job - source: modules/scheduler/batch-job-template - ... - -- id: batch-login - source: modules/scheduler/batch-login-node - use: [batch-job] - outputs: [instructions] -``` - -## Authentication - -To submit jobs from the login node, the service account attached to the VM needs -the `Batch Job Administrator` role. In most cases this service account will be -the Compute Engine default service account and will not be granted this role by -default. - -You can grant this role either by adding the `Batch Job Administrator` role to -the service account in the IAM page in the Google Cloud Console, or by running -the following command line: - -```bash -gcloud projects add-iam-policy-binding \ - --member=serviceAccount: \ - --role=roles/batch.jobsAdmin -``` - -## gcloud Batch Access - -Until the Google Cloud Batch API is generally available (GA), it may not be -available in all versions of the `gcloud` cli. You can test if the Google Cloud -Batch commands are available by running `gcloud [alpha|beta|] batch -h`. If the -Google Cloud Batch cli is not available it can generally be mitigated by either -updating `gcloud` by running `gcloud components update`, or using an image that -contains a more recent version of `gcloud`. - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 0.14.0 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [login\_startup\_script](#module\_login\_startup\_script) | ../../scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_compute_instance_from_template.batch_login](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_from_template) | resource | -| [google_compute_instance_template.batch_instance_template](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_instance_template) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [batch\_job\_directory](#input\_batch\_job\_directory) | The path of the directory on the login node in which to place the Google Cloud Batch job template | `string` | `"/home/batch-jobs"` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the deployment, also used for the job\_id | `string` | n/a | yes | -| [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | -| [gcloud\_version](#input\_gcloud\_version) | The version of the gcloud cli being used. Used for output instructions.
Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default
version). Typically supplied by a batch-job-template module. If multiple
batch-job-template modules supply the gcloud\_version, only the first will be used. | `string` | `""` | no | -| [instance\_template](#input\_instance\_template) | Login VM instance template self-link. Typically supplied by a
batch-job-template module. If multiple batch-job-template modules supply the
instance\_template, the first will be used. | `string` | n/a | yes | -| [job\_data](#input\_job\_data) | List of jobs and supporting data for each, typically provided via "use" from the batch-job-template module. | 
list(object({
    template_contents = string,
    filename          = string,
    id                = string
  }))
| n/a | yes | -| [job\_filename](#input\_job\_filename) | Deprecated (use `job_data`): The filename of the generated job template file. Typically supplied by a batch-job-template module. | `string` | `null` | no | -| [job\_id](#input\_job\_id) | Deprecated (use `job_data`): The ID for the Google Cloud Batch job. Typically supplied by a batch-job-template module for use in the output instructions. | `string` | `null` | no | -| [job\_template\_contents](#input\_job\_template\_contents) | Deprecated (use `job_data`): The contents of the Google Cloud Batch job template. Typically supplied by a batch-job-template module. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to the login node. Key-value pairs | `map(string)` | n/a | yes | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. Typically supplied by a batch-job-template module. | 
list(object({
    server_ip             = string
    remote_mount          = string
    local_mount           = string
    fs_type               = string
    mount_options         = string
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The region in which to create the login node | `string` | n/a | yes | -| [startup\_script](#input\_startup\_script) | Startup script run before Google Cloud Batch job starts. Typically supplied by a batch-job-template module. | `string` | `null` | no | -| [zone](#input\_zone) | The zone in which to create the login node | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [instructions](#output\_instructions) | Instructions for accessing the login node and submitting Google Cloud Batch jobs | -| [login\_node\_name](#output\_login\_node\_name) | Name of the created VM | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/main.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/main.tf deleted file mode 100644 index 6f539af122..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/main.tf +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "batch-login-node", ghpc_role = "scheduler" }) -} - -data "google_compute_instance_template" "batch_instance_template" { - name = var.instance_template -} - -locals { - job_template_runners = [for job in var.job_data : { - content = job.template_contents - destination = "${var.batch_job_directory}/${job.filename}" - type = "data" - }] - - instance_template_metadata = data.google_compute_instance_template.batch_instance_template.metadata - startup_metadata = { startup-script = module.login_startup_script.startup_script } - - oslogin_api_values = { - "DISABLE" = "FALSE" - "ENABLE" = "TRUE" - } - oslogin_metadata = var.enable_oslogin == "INHERIT" ? {} : { enable-oslogin = lookup(local.oslogin_api_values, var.enable_oslogin, "") } - - login_metadata = merge(local.instance_template_metadata, local.startup_metadata, local.oslogin_metadata) - - batch_command_instructions = join("\n", [for job in var.job_data : <<-EOT - ## For job: ${job.id} ## - - Submit your job from login node: - gcloud ${var.gcloud_version} batch jobs submit ${job.id} --config=${var.batch_job_directory}/${job.filename} --location=${var.region} --project=${var.project_id} - - Check status: - gcloud ${var.gcloud_version} batch jobs describe ${job.id} --location=${var.region} --project=${var.project_id} | grep state: - - Delete job: - gcloud ${var.gcloud_version} batch jobs delete ${job.id} --location=${var.region} --project=${var.project_id} - - EOT - ]) - - list_all_jobs = <<-EOT - List all jobs: - gcloud ${var.gcloud_version} batch jobs list --project=${var.project_id} - EOT - - readme_contents = <<-EOT - # Batch Job Templates - - This folder contains Batch job templates created by the Cluster Toolkit. - These templates can be edited before submitting to Batch to capture more - complex workloads. - - Use the following commands to: - ${local.list_all_jobs} - - ${local.batch_command_instructions} - EOT - - # Construct startup script for network storage - storage_client_install_runners = [ - for i, ns in var.network_storage : merge(ns.client_install_runner, { - destination = "${i}-${ns.client_install_runner.destination}" - }) if ns.client_install_runner != null - ] - mount_runners = [ - for i, ns in var.network_storage : merge(ns.mount_runner, { - destination = "${i}-${ns.mount_runner.destination}" - }) if ns.mount_runner != null - ] - - startup_script_runner = { - content = var.startup_script != null ? var.startup_script : "echo 'Batch job template had no startup script'" - destination = "passed_startup_script.sh" - type = "shell" - } -} - -module "login_startup_script" { - source = "../../scripts/startup-script" - labels = local.labels - project_id = var.project_id - deployment_name = var.deployment_name - region = var.region - runners = concat( - local.storage_client_install_runners, - local.mount_runners, - [local.startup_script_runner], - local.job_template_runners, - [ - { - content = local.readme_contents - destination = "${var.batch_job_directory}/README.md" - type = "data" - } - ] - ) -} - -resource "google_compute_instance_from_template" "batch_login" { - name = "${var.deployment_name}-batch-login" - source_instance_template = var.instance_template - project = var.project_id - zone = var.zone - metadata = local.login_metadata - - service_account { - scopes = ["https://www.googleapis.com/auth/cloud-platform"] - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml deleted file mode 100644 index 9af2319b4a..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/metadata.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - batch.googleapis.com - - compute.googleapis.com - - storage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/outputs.tf deleted file mode 100644 index ea8eccf8d5..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/outputs.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "login_node_name" { - description = "Name of the created VM" - value = google_compute_instance_from_template.batch_login.name -} - -output "instructions" { - description = "Instructions for accessing the login node and submitting Google Cloud Batch jobs" - value = <<-EOT - - Batch job template files will be placed on the Batch login node in the following directory: - ${var.batch_job_directory} - - Use the following commands to: - SSH into the login node: - gcloud compute ssh --zone ${google_compute_instance_from_template.batch_login.zone} ${google_compute_instance_from_template.batch_login.name} --project ${google_compute_instance_from_template.batch_login.project} - - ${local.list_all_jobs} - - ${local.batch_command_instructions} - EOT -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/variables.tf deleted file mode 100644 index 3b9caa7001..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/variables.tf +++ /dev/null @@ -1,151 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "deployment_name" { - description = "Name of the deployment, also used for the job_id" - type = string -} - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "region" { - description = "The region in which to create the login node" - type = string -} - -variable "zone" { - description = "The zone in which to create the login node" - type = string -} - -variable "labels" { - description = "Labels to add to the login node. Key-value pairs" - type = map(string) -} - -variable "instance_template" { - description = <<-EOT - Login VM instance template self-link. Typically supplied by a - batch-job-template module. If multiple batch-job-template modules supply the - instance_template, the first will be used. - EOT - type = string -} - -variable "network_storage" { - description = "An array of network attached storage mounts to be configured. Typically supplied by a batch-job-template module." - type = list(object({ - server_ip = string - remote_mount = string - local_mount = string - fs_type = string - mount_options = string - client_install_runner = map(string) - mount_runner = map(string) - })) - default = [] -} - -variable "startup_script" { - description = "Startup script run before Google Cloud Batch job starts. Typically supplied by a batch-job-template module." - type = string - default = null -} - -variable "job_data" { - description = "List of jobs and supporting data for each, typically provided via \"use\" from the batch-job-template module." - type = list(object({ - template_contents = string, - filename = string, - id = string - })) - validation { - condition = length(distinct([for job in var.job_data : job.filename])) == length(var.job_data) - error_message = "All filenames in var.job_data must be unique." - } - validation { - condition = length(distinct([for job in var.job_data : job.id])) == length(var.job_data) - error_message = "All job IDs in var.job_data must be unique." - } -} - -# tflint-ignore: terraform_unused_declarations -variable "job_template_contents" { - description = "Deprecated (use `job_data`): The contents of the Google Cloud Batch job template. Typically supplied by a batch-job-template module." - type = string - default = null - validation { - condition = var.job_template_contents == null - error_message = "job_template_contents is deprecated. Please use `job_data` instead." - } -} - -# tflint-ignore: terraform_unused_declarations -variable "job_filename" { - description = "Deprecated (use `job_data`): The filename of the generated job template file. Typically supplied by a batch-job-template module." - type = string - default = null - validation { - condition = var.job_filename == null - error_message = "job_filename is deprecated. Please use `job_data` instead." - } -} - -# tflint-ignore: terraform_unused_declarations -variable "job_id" { - description = "Deprecated (use `job_data`): The ID for the Google Cloud Batch job. Typically supplied by a batch-job-template module for use in the output instructions." - type = string - default = null - validation { - condition = var.job_id == null - error_message = "job_id is deprecated. Please use `job_data` instead." - } -} - -variable "gcloud_version" { - description = <<-EOT - The version of the gcloud cli being used. Used for output instructions. - Valid inputs are `\"alpha\"`, `\"beta\"` and \"\" (empty string for default - version). Typically supplied by a batch-job-template module. If multiple - batch-job-template modules supply the gcloud_version, only the first will be used. - EOT - type = string - default = "" - - validation { - condition = contains(["alpha", "beta", ""], var.gcloud_version) - error_message = "Allowed values for gcloud_version are 'alpha', 'beta', or '' (empty string)." - } -} - -variable "batch_job_directory" { - description = "The path of the directory on the login node in which to place the Google Cloud Batch job template" - type = string - default = "/home/batch-jobs" -} - -variable "enable_oslogin" { - description = "Enable or Disable OS Login with \"ENABLE\" or \"DISABLE\". Set to \"INHERIT\" to inherit project OS Login setting." - type = string - default = "ENABLE" - validation { - condition = var.enable_oslogin == null ? false : contains(["ENABLE", "DISABLE", "INHERIT"], var.enable_oslogin) - error_message = "Allowed string values for var.enable_oslogin are \"ENABLE\", \"DISABLE\", or \"INHERIT\"." - } -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/versions.tf deleted file mode 100644 index 5109db2aee..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/batch-login-node/versions.tf +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = ">= 3.83" - } - } - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:batch-login-node/v1.57.0" - } - - required_version = ">= 0.14.0" -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/README.md b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/README.md deleted file mode 100644 index cdf6a9a3cf..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/README.md +++ /dev/null @@ -1,215 +0,0 @@ -## Description - -This module creates a Google Kubernetes Engine -([GKE](https://cloud.google.com/kubernetes-engine)) cluster. - -> **_NOTE:_** This is an experimental module and the functionality and -> documentation will likely be updated in the near future. This module has only -> been tested in limited capacity. - -### Example - -The following example creates a GKE cluster and a VPC designed to work with GKE. -See [VPC Network](#vpc-network) section for more information about network -requirements. - -```yaml - - id: network1 - source: modules/network/vpc - settings: - subnetwork_name: gke-subnet - secondary_ranges: - gke-subnet: - - range_name: pods - ip_cidr_range: 10.4.0.0/14 - - range_name: services - ip_cidr_range: 10.0.32.0/20 - - - id: gke_cluster - source: modules/scheduler/gke-cluster - use: [network1] -``` - -Also see a full [GKE example blueprint](../../../examples/hpc-gke.yaml). - -### VPC Network - -This module is configured to create a -[VPC-native cluster](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips). -This means that alias IPs are used and that the subnetwork requires secondary -ranges for pods and services. In the example shown above these secondary ranges -are created in the VPC module. By default the `gke-cluster` module will look for -ranges with the names `pods` and `services`. These names can be configured using -the `pods_ip_range_name` and `services_ip_range_name` settings. - -### Multi-networking - -To [enable Multi-networking](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment), pass multivpc module to gke-cluster module as described in example below. Passing a multivpc module enables multi networking and [Dataplane V2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2?hl=en) on the cluster. - -```yaml - - id: network - source: modules/network/vpc - settings: - subnetwork_name: gke-subnet - secondary_ranges: - gke-subnet: - - range_name: pods - ip_cidr_range: 10.4.0.0/14 - - range_name: services - ip_cidr_range: 10.0.32.0/20 - - - id: multinetwork - source: modules/network/multivpc - settings: - network_name_prefix: multivpc-net - network_count: 8 - global_ip_address_range: 172.16.0.0/12 - subnetwork_cidr_suffix: 16 - - - id: gke-cluster - source: modules/scheduler/gke-cluster - use: [network, multinetwork] ## enables multi networking and Dataplane V2 on cluster - settings: - cluster_name: $(vars.deployment_name) -``` - -Find an example of multi networking in GKE [here](../../../examples/gke-a3-megagpu.yaml). - -### Cluster Limitations - -The current implementations has the following limitations: - -- Autopilot is disabled -- Auto-provisioning of new node pools is disabled -- Network policies are not supported -- General addon configuration is not supported -- Only regional cluster is supported - -## License - - -Copyright 2022 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | >= 6.16 | -| [google-beta](#requirement\_google-beta) | >= 6.16 | -| [kubernetes](#requirement\_kubernetes) | >= 2.36 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 6.16 | -| [google-beta](#provider\_google-beta) | >= 6.16 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | -| [workload\_identity](#module\_workload\_identity) | terraform-google-modules/kubernetes-engine/google//modules/workload-identity | ~> 34.0 | - -## Resources - -| Name | Type | -|------|------| -| [google-beta_google_container_cluster.gke_cluster](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_cluster) | resource | -| [google-beta_google_container_node_pool.system_node_pools](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_node_pool) | resource | -| [google-beta_google_container_engine_versions.version_prefix_filter](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/data-sources/google_container_engine_versions) | data source | -| [google_client_config.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks enables multi networking and creates relevat network objects on the cluster. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [authenticator\_security\_group](#input\_authenticator\_security\_group) | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| [autoscaling\_profile](#input\_autoscaling\_profile) | (Beta) Optimize for utilization or availability when deciding to remove nodes. Can be BALANCED or OPTIMIZE\_UTILIZATION. | `string` | `"OPTIMIZE_UTILIZATION"` | no | -| [cloud\_dns\_config](#input\_cloud\_dns\_config) | Configuration for Using Cloud DNS for GKE.

additive\_vpc\_scope\_dns\_domain: This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = "CLOUD\_DNS" and cluster\_dns\_scope = "CLUSTER\_SCOPE" must both be set as well.
cluster\_dns: Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS.
cluster\_dns\_scope: The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE.
cluster\_dns\_domain: The suffix used for all cluster service records. | 
object({
    additive_vpc_scope_dns_domain = optional(string)
    cluster_dns                   = optional(string, "PROVIDER_UNSPECIFIED")
    cluster_dns_scope             = optional(string, "DNS_SCOPE_UNSPECIFIED")
    cluster_dns_domain            = optional(string)
  })
| 
{
  "additive_vpc_scope_dns_domain": null,
  "cluster_dns": "PROVIDER_UNSPECIFIED",
  "cluster_dns_domain": null,
  "cluster_dns_scope": "DNS_SCOPE_UNSPECIFIED"
}
| no | -| [cluster\_availability\_type](#input\_cluster\_availability\_type) | Type of cluster availability. Possible values are: {REGIONAL, ZONAL} | `string` | `"REGIONAL"` | no | -| [cluster\_reference\_type](#input\_cluster\_reference\_type) | How the google\_container\_node\_pool.system\_node\_pools refers to the cluster. Possible values are: {SELF\_LINK, NAME} | `string` | `"SELF_LINK"` | no | -| [configure\_workload\_identity\_sa](#input\_configure\_workload\_identity\_sa) | When true, a kubernetes service account will be created and bound using workload identity to the service account used to create the cluster. | `bool` | `false` | no | -| [default\_max\_pods\_per\_node](#input\_default\_max\_pods\_per\_node) | The default maximum number of pods per node in this cluster. | `number` | `null` | no | -| [deletion\_protection](#input\_deletion\_protection) | "Determines if the cluster can be deleted by gcluster commands or not".
To delete a cluster provisioned with deletion\_protection set to true, you must first set it to false and apply the changes.
Then proceed with deletion as usual. | `bool` | `false` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment. Used in the GKE cluster name by default and can be configured with `prefix_with_deployment_name`. | `string` | n/a | yes | -| [enable\_dataplane\_v2](#input\_enable\_dataplane\_v2) | Enables [Dataplane v2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2). This setting is immutable on clusters. If null, will default to false unless using multi-networking, in which case it will default to true | `bool` | `null` | no | -| [enable\_dcgm\_monitoring](#input\_enable\_dcgm\_monitoring) | Enable GKE to collect DCGM metrics | `bool` | `false` | no | -| [enable\_external\_dns\_endpoint](#input\_enable\_external\_dns\_endpoint) | Allow [DNS-based approach](https://cloud.google.com/kubernetes-engine/docs/concepts/network-isolation#dns-based_endpoint) for accessing the GKE control plane.
Refer this [dedicated blog](https://cloud.google.com/blog/products/containers-kubernetes/new-dns-based-endpoint-for-the-gke-control-plane) for more details. | `bool` | `false` | no | -| [enable\_filestore\_csi](#input\_enable\_filestore\_csi) | The status of the Filestore Container Storage Interface (CSI) driver addon, which allows the usage of filestore instance as volumes. | `bool` | `false` | no | -| [enable\_gcsfuse\_csi](#input\_enable\_gcsfuse\_csi) | The status of the GCSFuse Container Storage Interface (CSI) driver addon, which allows the usage of a GCS bucket as volumes. | `bool` | `false` | no | -| [enable\_k8s\_beta\_apis](#input\_enable\_k8s\_beta\_apis) | List of Enabled Kubernetes Beta APIs. | `list(string)` | `null` | no | -| [enable\_master\_global\_access](#input\_enable\_master\_global\_access) | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `false` | no | -| [enable\_multi\_networking](#input\_enable\_multi\_networking) | Enables [multi networking](https://cloud.google.com/kubernetes-engine/docs/how-to/setup-multinetwork-support-for-pods#create-a-gke-cluster) (Requires GKE Enterprise). This setting is immutable on clusters and enables [Dataplane V2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2?hl=en). If null, will determine state based on if additional\_networks are passed in. | `bool` | `null` | no | -| [enable\_node\_local\_dns\_cache](#input\_enable\_node\_local\_dns\_cache) | Enable GKE NodeLocal DNSCache addon to improve DNS lookup latency | `bool` | `false` | no | -| [enable\_parallelstore\_csi](#input\_enable\_parallelstore\_csi) | The status of the Google Compute Engine Parallelstore Container Storage Interface (CSI) driver addon, which allows the usage of a parallelstore as volumes. | `bool` | `false` | no | -| [enable\_persistent\_disk\_csi](#input\_enable\_persistent\_disk\_csi) | The status of the Google Compute Engine Persistent Disk Container Storage Interface (CSI) driver addon, which allows the usage of a PD as volumes. | `bool` | `true` | no | -| [enable\_private\_endpoint](#input\_enable\_private\_endpoint) | (Beta) Whether the master's internal IP address is used as the cluster endpoint. | `bool` | `true` | no | -| [enable\_private\_ipv6\_google\_access](#input\_enable\_private\_ipv6\_google\_access) | The private IPv6 google access type for the VMs in this subnet. | `bool` | `true` | no | -| [enable\_private\_nodes](#input\_enable\_private\_nodes) | (Beta) Whether nodes have internal IP addresses only. | `bool` | `true` | no | -| [enable\_ray\_operator](#input\_enable\_ray\_operator) | The status of the Ray operator addon, This feature enables Kubernetes APIs for managing and scaling Ray clusters and jobs. You control and are responsible for managing ray.io custom resources in your cluster. This feature is not compatible with GKE clusters that already have another Ray operator installed. Supports clusters on Kubernetes version 1.29.8-gke.1054000 or later. | `bool` | `false` | no | -| [gcp\_public\_cidrs\_access\_enabled](#input\_gcp\_public\_cidrs\_access\_enabled) | Whether the cluster master is accessible via all the Google Compute Engine Public IPs. To view this list of IP addresses look here https://cloud.google.com/compute/docs/faq#find_ip_range | `bool` | `false` | no | -| [k8s\_network\_names](#input\_k8s\_network\_names) | Kubernetes network names details for GKE. If starting index is not specified for gvnic or rdma, it would be set to the default values. | 
object({
    gvnic_prefix      = optional(string, "")
    gvnic_start_index = optional(number, 1)
    gvnic_postfix     = optional(string, "")
    rdma_prefix       = optional(string, "")
    rdma_start_index  = optional(number, 0)
    rdma_postfix      = optional(string, "")
  })
| 
{
  "gvnic_postfix": "",
  "gvnic_prefix": "gvnic-",
  "gvnic_start_index": 1,
  "rdma_postfix": "",
  "rdma_prefix": "rdma-",
  "rdma_start_index": 0
}
| no | -| [k8s\_service\_account\_name](#input\_k8s\_service\_account\_name) | Kubernetes service account name to use with the gke cluster | `string` | `"workload-identity-k8s-sa"` | no | -| [labels](#input\_labels) | GCE resource labels to be applied to resources. Key-value pairs. | `map(string)` | n/a | yes | -| [maintenance\_exclusions](#input\_maintenance\_exclusions) | List of maintenance exclusions. A cluster can have up to three. | 
list(object({
    name            = string
    start_time      = string
    end_time        = string
    exclusion_scope = string
  }))
| `[]` | no | -| [maintenance\_start\_time](#input\_maintenance\_start\_time) | Start time for daily maintenance operations. Specified in GMT with `HH:MM` format. | `string` | `"09:00"` | no | -| [master\_authorized\_networks](#input\_master\_authorized\_networks) | External network that can access Kubernetes master through HTTPS. Must be specified in CIDR notation. | 
list(object({
    cidr_block   = string
    display_name = string
  }))
| `[]` | no | -| [master\_ipv4\_cidr\_block](#input\_master\_ipv4\_cidr\_block) | (Beta) The IP range in CIDR notation to use for the hosted master network. | `string` | `"172.16.0.32/28"` | no | -| [min\_master\_version](#input\_min\_master\_version) | The minimum version of the master. If unset, the cluster's version will be set by GKE to the version of the most recent official release. | `string` | `null` | no | -| [name\_suffix](#input\_name\_suffix) | Custom cluster name postpended to the `deployment_name`. See `prefix_with_deployment_name`. | `string` | `""` | no | -| [network\_id](#input\_network\_id) | The ID of the GCE VPC network to host the cluster given in the format: `projects//global/networks/`. | `string` | n/a | yes | -| [networking\_mode](#input\_networking\_mode) | Determines whether alias IPs or routes will be used for pod IPs in the cluster. Options are VPC\_NATIVE or ROUTES. VPC\_NATIVE enables IP aliasing. The default is VPC\_NATIVE. | `string` | `"VPC_NATIVE"` | no | -| [pods\_ip\_range\_name](#input\_pods\_ip\_range\_name) | The name of the secondary subnet ip range to use for pods. | `string` | `"pods"` | no | -| [prefix\_with\_deployment\_name](#input\_prefix\_with\_deployment\_name) | If true, cluster name will be prefixed by `deployment_name` (ex: -). | `bool` | `true` | no | -| [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | -| [region](#input\_region) | The region to host the cluster in. | `string` | n/a | yes | -| [release\_channel](#input\_release\_channel) | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. | `string` | `"UNSPECIFIED"` | no | -| [service\_account](#input\_service\_account) | DEPRECATED: use service\_account\_email and scopes. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | -| [service\_account\_email](#input\_service\_account\_email) | Service account e-mail address to use with the system node pool | `string` | `null` | no | -| [service\_account\_scopes](#input\_service\_account\_scopes) | Scopes to to use with the system node pool. | `set(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | -| [services\_ip\_range\_name](#input\_services\_ip\_range\_name) | The name of the secondary subnet range to use for services. | `string` | `"services"` | no | -| [subnetwork\_self\_link](#input\_subnetwork\_self\_link) | The self link of the subnetwork to host the cluster in. | `string` | n/a | yes | -| [system\_node\_pool\_disk\_size\_gb](#input\_system\_node\_pool\_disk\_size\_gb) | Size of disk for each node of the system node pool. | `number` | `100` | no | -| [system\_node\_pool\_disk\_type](#input\_system\_node\_pool\_disk\_type) | Disk type for each node of the system node pool. | `string` | `null` | no | -| [system\_node\_pool\_enable\_secure\_boot](#input\_system\_node\_pool\_enable\_secure\_boot) | Enable secure boot for the nodes. Keep enabled unless custom kernel modules need to be loaded. See [here](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#secure-boot) for more info. | `bool` | `true` | no | -| [system\_node\_pool\_enabled](#input\_system\_node\_pool\_enabled) | Create a system node pool. | `bool` | `true` | no | -| [system\_node\_pool\_image\_type](#input\_system\_node\_pool\_image\_type) | The default image type used by NAP once a new node pool is being created. Use either COS\_CONTAINERD or UBUNTU\_CONTAINERD. | `string` | `"COS_CONTAINERD"` | no | -| [system\_node\_pool\_kubernetes\_labels](#input\_system\_node\_pool\_kubernetes\_labels) | Kubernetes labels to be applied to each node in the node group. Key-value pairs.
(The `kubernetes.io/` and `k8s.io/` prefixes are reserved by Kubernetes Core components and cannot be specified) | `map(string)` | `null` | no | -| [system\_node\_pool\_machine\_type](#input\_system\_node\_pool\_machine\_type) | Machine type for the system node pool. | `string` | `"e2-standard-4"` | no | -| [system\_node\_pool\_name](#input\_system\_node\_pool\_name) | Name of the system node pool. | `string` | `"system"` | no | -| [system\_node\_pool\_node\_count](#input\_system\_node\_pool\_node\_count) | The total min and max nodes to be maintained in the system node pool. | 
object({
    total_min_nodes = number
    total_max_nodes = number
  })
| 
{
  "total_max_nodes": 10,
  "total_min_nodes": 2
}
| no | -| [system\_node\_pool\_taints](#input\_system\_node\_pool\_taints) | Taints to be applied to the system node pool. | 
list(object({
    key    = string
    value  = any
    effect = string
  }))
| 
[
  {
    "effect": "NO_SCHEDULE",
    "key": "components.gke.io/gke-managed-components",
    "value": true
  }
]
| no | -| [system\_node\_pool\_zones](#input\_system\_node\_pool\_zones) | The zones to use for the system node pool. If not specified, the cluster default node zone(s) will be used. | `list(string)` | `null` | no | -| [timeout\_create](#input\_timeout\_create) | Timeout for creating a node pool | `string` | `null` | no | -| [timeout\_update](#input\_timeout\_update) | Timeout for updating a node pool | `string` | `null` | no | -| [upgrade\_settings](#input\_upgrade\_settings) | Defines gke cluster upgrade settings. It is highly recommended that you define all max\_surge and max\_unavailable.
If max\_surge is not specified, it would be set to a default value of 0.
If max\_unavailable is not specified, it would be set to a default value of 1. | 
object({
    strategy        = string
    max_surge       = optional(number)
    max_unavailable = optional(number)
  })
| 
{
  "max_surge": 0,
  "max_unavailable": 1,
  "strategy": "SURGE"
}
| no | -| [version\_prefix](#input\_version\_prefix) | If provided, Terraform will only return versions that match the string prefix. For example, `1.31.` will match all `1.31` series releases. Since this is just a string match, it's recommended that you append a `.` after minor versions to ensure that prefixes such as `1.3` don't match versions like `1.30.1-gke.10` accidentally. | `string` | `"1.31."` | no | -| [zone](#input\_zone) | Zone for a zonal cluster. | `string` | `null` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [cluster\_id](#output\_cluster\_id) | An identifier for the resource with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | -| [gke\_cluster\_exists](#output\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations. | -| [gke\_version](#output\_gke\_version) | GKE cluster's version. | -| [instructions](#output\_instructions) | Instructions on how to connect to the created cluster. | -| [k8s\_service\_account\_name](#output\_k8s\_service\_account\_name) | Name of k8s service account. | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/main.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/main.tf deleted file mode 100644 index 0ec1fde35b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/main.tf +++ /dev/null @@ -1,447 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "gke-cluster", ghpc_role = "scheduler" }) -} - -locals { - upgrade_settings = { - strategy = var.upgrade_settings.strategy - max_surge = coalesce(var.upgrade_settings.max_surge, 0) - max_unavailable = coalesce(var.upgrade_settings.max_unavailable, 1) - } -} - -locals { - dash = var.prefix_with_deployment_name && var.name_suffix != "" ? "-" : "" - prefix = var.prefix_with_deployment_name ? var.deployment_name : "" - name_maybe_empty = "${local.prefix}${local.dash}${var.name_suffix}" - name = local.name_maybe_empty != "" ? local.name_maybe_empty : "NO-NAME-GIVEN" - - cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ - security_group = var.authenticator_security_group - }] - - default_sa_email = "${data.google_project.project.number}-compute@developer.gserviceaccount.com" - sa_email = coalesce(var.service_account_email, local.default_sa_email) - - # additional VPCs enable multi networking - derived_enable_multi_networking = coalesce(var.enable_multi_networking, length(var.additional_networks) > 0) - - # multi networking needs enabled Dataplane v2 - derived_enable_dataplane_v2 = coalesce(var.enable_dataplane_v2, local.derived_enable_multi_networking) - - default_monitoring_component = [ - "SYSTEM_COMPONENTS", - "POD", - "DAEMONSET", - "DEPLOYMENT", - "STATEFULSET", - "STORAGE", - "HPA", - "CADVISOR", - "KUBELET" - ] - - default_logging_component = [ - "SYSTEM_COMPONENTS", - "WORKLOADS" - ] -} - -data "google_project" "project" { - project_id = var.project_id -} - -data "google_container_engine_versions" "version_prefix_filter" { - provider = google-beta - location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region - version_prefix = var.version_prefix -} - -locals { - master_version = var.min_master_version != null ? var.min_master_version : data.google_container_engine_versions.version_prefix_filter.latest_master_version -} - -resource "google_container_cluster" "gke_cluster" { - provider = google-beta - - project = var.project_id - name = local.name - location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region - resource_labels = local.labels - networking_mode = var.networking_mode - # decouple node pool lifecycle from cluster life cycle - remove_default_node_pool = true - initial_node_count = 1 # must be set when remove_default_node_pool is set - node_locations = var.system_node_pool_zones - - deletion_protection = var.deletion_protection - - dynamic "enable_k8s_beta_apis" { - for_each = var.enable_k8s_beta_apis != null ? [1] : [] - content { - enabled_apis = var.enable_k8s_beta_apis - } - } - - network = var.network_id - subnetwork = var.subnetwork_self_link - - # Note: the existence of the "master_authorized_networks_config" block enables - # the master authorized networks even if it's empty. - master_authorized_networks_config { - dynamic "cidr_blocks" { - for_each = var.master_authorized_networks - content { - cidr_block = cidr_blocks.value.cidr_block - display_name = cidr_blocks.value.display_name - } - } - gcp_public_cidrs_access_enabled = var.gcp_public_cidrs_access_enabled - } - - private_ipv6_google_access = var.enable_private_ipv6_google_access ? "PRIVATE_IPV6_GOOGLE_ACCESS_TO_GOOGLE" : null - default_max_pods_per_node = var.default_max_pods_per_node - master_auth { - client_certificate_config { - issue_client_certificate = false - } - } - - enable_shielded_nodes = true - - cluster_autoscaling { - # Controls auto provisioning of node-pools - enabled = false - - # Controls autoscaling algorithm of node-pools - autoscaling_profile = var.autoscaling_profile - } - - datapath_provider = local.derived_enable_dataplane_v2 ? "ADVANCED_DATAPATH" : "LEGACY_DATAPATH" - - enable_multi_networking = local.derived_enable_multi_networking - - network_policy { - # Enabling NetworkPolicy for clusters with DatapathProvider=ADVANCED_DATAPATH - # is not allowed. Dataplane V2 will take care of network policy enforcement - # instead. - enabled = false - # GKE Dataplane V2 support. This must be set to PROVIDER_UNSPECIFIED in - # order to let the datapath_provider take effect. - # https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/656#issuecomment-720398658 - provider = "PROVIDER_UNSPECIFIED" - } - - private_cluster_config { - enable_private_nodes = var.enable_private_nodes - enable_private_endpoint = var.enable_private_endpoint - master_ipv4_cidr_block = var.master_ipv4_cidr_block - master_global_access_config { - enabled = var.enable_master_global_access - } - } - - ip_allocation_policy { - cluster_secondary_range_name = var.pods_ip_range_name - services_secondary_range_name = var.services_ip_range_name - } - - workload_identity_config { - workload_pool = "${var.project_id}.svc.id.goog" - } - - dynamic "authenticator_groups_config" { - for_each = local.cluster_authenticator_security_group - content { - security_group = authenticator_groups_config.value.security_group - } - } - - release_channel { - channel = var.release_channel - } - min_master_version = local.master_version - - maintenance_policy { - daily_maintenance_window { - start_time = var.maintenance_start_time - } - - dynamic "maintenance_exclusion" { - for_each = var.maintenance_exclusions - content { - exclusion_name = maintenance_exclusion.value.name - start_time = maintenance_exclusion.value.start_time - end_time = maintenance_exclusion.value.end_time - exclusion_options { - scope = maintenance_exclusion.value.exclusion_scope - } - } - } - } - - dynamic "dns_config" { - for_each = var.cloud_dns_config != null ? [1] : [] - content { - additive_vpc_scope_dns_domain = var.cloud_dns_config.additive_vpc_scope_dns_domain - cluster_dns = var.cloud_dns_config.cluster_dns - cluster_dns_scope = var.cloud_dns_config.cluster_dns_scope - cluster_dns_domain = var.cloud_dns_config.cluster_dns_domain - } - } - - addons_config { - gcp_filestore_csi_driver_config { - enabled = var.enable_filestore_csi - } - gcs_fuse_csi_driver_config { - enabled = var.enable_gcsfuse_csi - } - gce_persistent_disk_csi_driver_config { - enabled = var.enable_persistent_disk_csi - } - dns_cache_config { - enabled = var.enable_node_local_dns_cache - } - parallelstore_csi_driver_config { - enabled = var.enable_parallelstore_csi - } - ray_operator_config { - enabled = var.enable_ray_operator - } - } - - timeouts { - create = var.timeout_create - update = var.timeout_update - } - - node_config { - shielded_instance_config { - enable_secure_boot = var.system_node_pool_enable_secure_boot - enable_integrity_monitoring = true - } - } - - control_plane_endpoints_config { - dns_endpoint_config { - allow_external_traffic = var.enable_external_dns_endpoint - } - } - - lifecycle { - # Ignore all changes to the default node pool. It's being removed after creation. - ignore_changes = [ - node_config, - min_master_version, - ] - precondition { - condition = var.default_max_pods_per_node == null || var.networking_mode == "VPC_NATIVE" - error_message = "default_max_pods_per_node does not work on `routes-based` clusters, that don't have IP Aliasing enabled." - } - precondition { - condition = coalesce(var.enable_dataplane_v2, true) || !local.derived_enable_multi_networking - error_message = "'enable_dataplane_v2' cannot be false when enabling multi networking." - } - precondition { - condition = coalesce(var.enable_multi_networking, true) || length(var.additional_networks) == 0 - error_message = "'enable_multi_networking' cannot be false when using multivpc module, which passes additional_networks." - } - } - - monitoring_config { - enable_components = var.enable_dcgm_monitoring ? concat(local.default_monitoring_component, ["DCGM"]) : local.default_monitoring_component - managed_prometheus { - enabled = true - } - } - - logging_config { - enable_components = local.default_logging_component - } -} - -# We define explicit node pools, so that it can be modified without -# having to destroy the entire cluster. -resource "google_container_node_pool" "system_node_pools" { - provider = google-beta - count = var.system_node_pool_enabled ? 1 : 0 - - project = var.project_id - name = var.system_node_pool_name - cluster = var.cluster_reference_type == "NAME" ? google_container_cluster.gke_cluster.name : google_container_cluster.gke_cluster.self_link - location = var.cluster_availability_type == "ZONAL" ? var.zone : var.region - node_locations = var.system_node_pool_zones - version = local.master_version - - autoscaling { - total_min_node_count = var.system_node_pool_node_count.total_min_nodes - total_max_node_count = var.system_node_pool_node_count.total_max_nodes - } - - upgrade_settings { - strategy = local.upgrade_settings.strategy - max_surge = local.upgrade_settings.max_surge - max_unavailable = local.upgrade_settings.max_unavailable - } - - management { - auto_repair = true - auto_upgrade = true - } - - node_config { - labels = var.system_node_pool_kubernetes_labels - resource_labels = local.labels - service_account = var.service_account_email - oauth_scopes = var.service_account_scopes - machine_type = var.system_node_pool_machine_type - disk_size_gb = var.system_node_pool_disk_size_gb - disk_type = var.system_node_pool_disk_type - - dynamic "taint" { - for_each = var.system_node_pool_taints - content { - key = taint.value.key - value = taint.value.value - effect = taint.value.effect - } - } - - # Forcing the use of the Container-optimized image, as it is the only - # image with the proper logging daemon installed. - # - # cos images use Shielded VMs since v1.13.6-gke.0. - # https://cloud.google.com/kubernetes-engine/docs/how-to/node-images - # - # We use COS_CONTAINERD to be compatible with (optional) gVisor. - # https://cloud.google.com/kubernetes-engine/docs/how-to/sandbox-pods - image_type = var.system_node_pool_image_type - - shielded_instance_config { - enable_secure_boot = var.system_node_pool_enable_secure_boot - enable_integrity_monitoring = true - } - - gvnic { - enabled = var.system_node_pool_image_type == "COS_CONTAINERD" - } - - # Implied by Workload Identity - workload_metadata_config { - mode = "GKE_METADATA" - } - # Implied by workload identity. - metadata = { - "disable-legacy-endpoints" = "true" - } - } - - lifecycle { - ignore_changes = [ - node_config[0].labels, - node_config[0].taint, - version, - ] - precondition { - condition = contains(["SURGE"], local.upgrade_settings.strategy) - error_message = "Only SURGE strategy is supported" - } - precondition { - condition = local.upgrade_settings.max_unavailable >= 0 - error_message = "max_unavailable should be set to 0 or greater" - } - precondition { - condition = local.upgrade_settings.max_surge >= 0 - error_message = "max_surge should be set to 0 or greater" - } - precondition { - condition = local.upgrade_settings.max_unavailable > 0 || local.upgrade_settings.max_surge > 0 - error_message = "At least one of max_unavailable or max_surge must greater than 0" - } - } -} - -data "google_client_config" "default" {} - -provider "kubernetes" { - host = "https://${google_container_cluster.gke_cluster.endpoint}" - cluster_ca_certificate = base64decode(google_container_cluster.gke_cluster.master_auth[0].cluster_ca_certificate) - token = data.google_client_config.default.access_token -} - -module "workload_identity" { - count = var.configure_workload_identity_sa ? 1 : 0 - source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity" - version = "~> 34.0" - - use_existing_gcp_sa = true - name = var.k8s_service_account_name - gcp_sa_name = local.sa_email - project_id = var.project_id - - # https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1059 - depends_on = [ - data.google_project.project, - google_container_cluster.gke_cluster - ] -} - -locals { - k8s_service_account_name = one(module.workload_identity[*].k8s_service_account_name) -} - -locals { - # Separate gvnic and rdma networks and assign indexes - gvnic_networks = [for idx, net in [for n in var.additional_networks : n if strcontains(upper(n.nic_type), "GVNIC")] : - merge(net, { name = "${var.k8s_network_names.gvnic_prefix}${idx + var.k8s_network_names.gvnic_start_index}${var.k8s_network_names.gvnic_postfix}" }) - ] - - rdma_networks = [for idx, net in [for n in var.additional_networks : n if strcontains(upper(n.nic_type), "RDMA")] : - merge(net, { name = "${var.k8s_network_names.rdma_prefix}${idx + var.k8s_network_names.rdma_start_index}${var.k8s_network_names.rdma_postfix}" }) - ] - - all_networks = concat(local.gvnic_networks, local.rdma_networks) -} - -module "kubectl_apply" { - source = "../../management/kubectl-apply" - - cluster_id = google_container_cluster.gke_cluster.id - project_id = var.project_id - - apply_manifests = flatten([ - for idx, network_info in local.all_networks : [ - { - source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", - template_vars = { - name = network_info.name, - network_name = network_info.network - subnetwork_name = network_info.subnetwork, - device_mode = strcontains(upper(network_info.nic_type), "RDMA") ? "RDMA" : "NetDevice" - } - }, - { - source = "${path.module}/templates/network-object.yaml.tftpl", - template_vars = { name = network_info.name } - } - ] - ]) -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml deleted file mode 100644 index bd1517ce8f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/outputs.tf deleted file mode 100644 index 3326a5468e..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/outputs.tf +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "cluster_id" { - description = "An identifier for the resource with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." - value = google_container_cluster.gke_cluster.id -} - -output "gke_cluster_exists" { - description = "A static flag that signals to downstream modules that a cluster has been created. Needed by community/modules/scripts/kubernetes-operations." - value = true - depends_on = [ - google_container_cluster.gke_cluster - ] -} - -locals { - private_endpoint_message = trimspace( - <<-EOT - This cluster was created with 'enable_private_endpoint: true'. - It cannot be accessed from a public IP addresses. - One way to access this cluster is from a VM created in the GKE cluster subnet. - EOT - ) - master_authorized_networks_message = length(var.master_authorized_networks) == 0 ? "" : trimspace( - <<-EOT - The following networks have been authorized to access this cluster: - ${join("\n", [for x in var.master_authorized_networks : " ${x.display_name}: ${x.cidr_block}"])}" - EOT - ) - public_endpoint_message = trimspace( - <<-EOT - To add authorized networks you can allowlist your IP with this command: - gcloud container clusters update ${google_container_cluster.gke_cluster.name} \ - --region ${google_container_cluster.gke_cluster.location} \ - --project ${var.project_id} \ - --enable-master-authorized-networks \ - --master-authorized-networks /32 - EOT - ) - allowlist_your_ip_message = var.enable_private_endpoint ? local.private_endpoint_message : local.public_endpoint_message - kubernetes_service_account_message = local.k8s_service_account_name == null ? "" : trimspace( - <<-EOT - Use the following Kubernetes Service Account in the default namespace to run your workloads: - ${local.k8s_service_account_name} - The GCP Service Account mapped to this Kubernetes Service Account is: - ${local.sa_email} - EOT - ) - kubernetes_cluster_fetch_credential_message = var.enable_external_dns_endpoint ? trimspace( - <<-EOT - Use the following command to fetch credentials for the created cluster: - gcloud container clusters get-credentials ${google_container_cluster.gke_cluster.name} \ - --region ${google_container_cluster.gke_cluster.location} \ - --project ${var.project_id} \ - --dns-endpoint - EOT - ) : trimspace( - <<-EOT - Use the following command to fetch credentials for the created cluster: - gcloud container clusters get-credentials ${google_container_cluster.gke_cluster.name} \ - --region ${google_container_cluster.gke_cluster.location} \ - --project ${var.project_id} - EOT - ) -} - -output "instructions" { - description = "Instructions on how to connect to the created cluster." - value = trimspace( - <<-EOT - ${local.master_authorized_networks_message} - - ${local.allowlist_your_ip_message} - - ${local.kubernetes_cluster_fetch_credential_message} - - ${local.kubernetes_service_account_message} - EOT - ) -} - -output "k8s_service_account_name" { - description = "Name of k8s service account." - value = local.k8s_service_account_name -} - -output "gke_version" { - description = "GKE cluster's version." - value = google_container_cluster.gke_cluster.master_version -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl deleted file mode 100644 index d376a1a760..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: networking.gke.io/v1 -kind: GKENetworkParamSet -metadata: - name: ${name} -spec: - vpc: ${network_name} - vpcSubnet: ${subnetwork_name} - deviceMode: ${device_mode} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl deleted file mode 100644 index 1571a92692..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/templates/network-object.yaml.tftpl +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: networking.gke.io/v1 -kind: Network -metadata: - name: ${name} -spec: - parametersRef: - group: networking.gke.io - kind: GKENetworkParamSet - name: ${name} - type: Device diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/variables.tf deleted file mode 100644 index 7dcb9d0ef0..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/gke-cluster/variables.tf +++ /dev/null @@ -1,521 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -variable "project_id" { - description = "The project ID to host the cluster in." - type = string -} - -variable "name_suffix" { - description = "Custom cluster name postpended to the `deployment_name`. See `prefix_with_deployment_name`." - type = string - default = "" -} - -variable "deployment_name" { - description = "Name of the HPC deployment. Used in the GKE cluster name by default and can be configured with `prefix_with_deployment_name`." - type = string -} - -variable "prefix_with_deployment_name" { - description = "If true, cluster name will be prefixed by `deployment_name` (ex: -)." - type = bool - default = true -} - -variable "region" { - description = "The region to host the cluster in." - type = string -} - -variable "zone" { - description = "Zone for a zonal cluster." - default = null - type = string -} - -variable "network_id" { - description = "The ID of the GCE VPC network to host the cluster given in the format: `projects//global/networks/`." - type = string - validation { - condition = length(split("/", var.network_id)) == 5 - error_message = "The network id must be provided in the following format: projects//global/networks/." - } -} - -variable "subnetwork_self_link" { - description = "The self link of the subnetwork to host the cluster in." - type = string -} - -variable "pods_ip_range_name" { - description = "The name of the secondary subnet ip range to use for pods." - type = string - default = "pods" -} - -variable "services_ip_range_name" { - description = "The name of the secondary subnet range to use for services." - type = string - default = "services" -} - -variable "enable_private_ipv6_google_access" { - description = "The private IPv6 google access type for the VMs in this subnet." - type = bool - default = true -} - -variable "release_channel" { - description = "The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`." - type = string - default = "UNSPECIFIED" -} - -variable "min_master_version" { - description = "The minimum version of the master. If unset, the cluster's version will be set by GKE to the version of the most recent official release." - type = string - default = null -} - -variable "version_prefix" { - description = "If provided, Terraform will only return versions that match the string prefix. For example, `1.31.` will match all `1.31` series releases. Since this is just a string match, it's recommended that you append a `.` after minor versions to ensure that prefixes such as `1.3` don't match versions like `1.30.1-gke.10` accidentally." - type = string - default = "1.31." -} - -variable "maintenance_start_time" { - description = "Start time for daily maintenance operations. Specified in GMT with `HH:MM` format." - type = string - default = "09:00" -} - -variable "maintenance_exclusions" { - description = "List of maintenance exclusions. A cluster can have up to three." - type = list(object({ - name = string - start_time = string - end_time = string - exclusion_scope = string - })) - default = [] - validation { - condition = alltrue([ - for x in var.maintenance_exclusions : - contains(["NO_UPGRADES", "NO_MINOR_UPGRADES", "NO_MINOR_OR_NODE_UPGRADES"], x.exclusion_scope) - ]) - error_message = "`exclusion_scope` must be set to `NO_UPGRADES` OR `NO_MINOR_UPGRADES` OR `NO_MINOR_OR_NODE_UPGRADES`." - } -} - -variable "cloud_dns_config" { - description = < **_NOTE:_** The `project_id` and `region` settings would be inferred from the -> deployment variables of the same name, but they are included here for clarity. - -### Multi-networking - -To create network objects in GKE cluster, you can pass a multivpc module to a pre-existing-gke-cluster module instead of [applying a manifest manually](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx#create-gke-environment). - -```yaml - - id: network - source: modules/network/vpc - - - id: multinetwork - source: modules/network/multivpc - settings: - network_name_prefix: multivpc-net - network_count: 8 - global_ip_address_range: 172.16.0.0/12 - subnetwork_cidr_suffix: 16 - - - id: existing-gke-cluster ## multinetworking must be enabled in advance when cluster creation - source: modules/scheduler/pre-existing-gke-cluster - use: [multinetwork] - settings: - cluster_name: $(vars.deployment_name) -``` - -## License - - -Copyright 2024 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3 | -| [google](#requirement\_google) | > 5.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | > 5.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [kubectl\_apply](#module\_kubectl\_apply) | ../../management/kubectl-apply | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_container_cluster.existing_gke_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/container_cluster) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks creates relevat network objects on the cluster. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | -| [cluster\_name](#input\_cluster\_name) | Name of the existing cluster | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Project that hosts the existing cluster | `string` | n/a | yes | -| [rdma\_subnetwork\_name\_prefix](#input\_rdma\_subnetwork\_name\_prefix) | Prefix of the RDMA subnetwork names | `string` | `null` | no | -| [region](#input\_region) | Region in which to search for the cluster | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [cluster\_id](#output\_cluster\_id) | An identifier for the gke cluster with format projects/{{project\_id}}/locations/{{region}}/clusters/{{name}}. | -| [gke\_cluster\_exists](#output\_gke\_cluster\_exists) | A static flag that signals to downstream modules that a cluster exists. | -| [gke\_version](#output\_gke\_version) | GKE cluster's version. | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf deleted file mode 100644 index 926d2be100..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/main.tf +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -data "google_container_cluster" "existing_gke_cluster" { - name = var.cluster_name - project = var.project_id - location = var.region -} - -locals { - rdma_networks = [for network_info in var.additional_networks : network_info if strcontains(upper(network_info.nic_type), "RDMA")] - non_rdma_networks = [for network_info in var.additional_networks : network_info if !strcontains(upper(network_info.nic_type), "RDMA")] - apply_manifests_rdma_networks = flatten([ - for idx, network_info in local.rdma_networks : [ - { - source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", - template_vars = { - name = "${var.rdma_subnetwork_name_prefix}-${idx}", - network_name = network_info.network - subnetwork_name = "${var.rdma_subnetwork_name_prefix}-${idx}", - device_mode = "RDMA" - } - }, - { - source = "${path.module}/templates/network-object.yaml.tftpl", - template_vars = { name = "${var.rdma_subnetwork_name_prefix}-${idx}" } - } - ] - ]) - - apply_manifests_non_rdma_networks = flatten([ - for idx, network_info in local.non_rdma_networks : [ - { - source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", - template_vars = { - name = network_info.subnetwork - network_name = network_info.network - subnetwork_name = network_info.subnetwork - device_mode = "NetDevice" - } - }, - { - source = "${path.module}/templates/network-object.yaml.tftpl", - template_vars = { name = network_info.subnetwork } - } - ] - ]) -} - -module "kubectl_apply" { - source = "../../management/kubectl-apply" - - cluster_id = data.google_container_cluster.existing_gke_cluster.id - project_id = var.project_id - - apply_manifests = concat(local.apply_manifests_non_rdma_networks, local.apply_manifests_rdma_networks) -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml deleted file mode 100644 index 17bedb471b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - container.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf deleted file mode 100644 index 8884ee30b0..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/outputs.tf +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "cluster_id" { - description = "An identifier for the gke cluster with format projects/{{project_id}}/locations/{{region}}/clusters/{{name}}." - value = data.google_container_cluster.existing_gke_cluster.id -} - -output "gke_cluster_exists" { - description = "A static flag that signals to downstream modules that a cluster exists." - value = true - depends_on = [ - data.google_container_cluster.existing_gke_cluster - ] -} - -output "gke_version" { - description = "GKE cluster's version." - value = data.google_container_cluster.existing_gke_cluster.master_version -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl deleted file mode 100644 index d376a1a760..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: networking.gke.io/v1 -kind: GKENetworkParamSet -metadata: - name: ${name} -spec: - vpc: ${network_name} - vpcSubnet: ${subnetwork_name} - deviceMode: ${device_mode} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl deleted file mode 100644 index 1571a92692..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/templates/network-object.yaml.tftpl +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: networking.gke.io/v1 -kind: Network -metadata: - name: ${name} -spec: - parametersRef: - group: networking.gke.io - kind: GKENetworkParamSet - name: ${name} - type: Device diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf deleted file mode 100644 index 9e9ed98ed3..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/variables.tf +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project that hosts the existing cluster" - type = string -} - -variable "cluster_name" { - description = "Name of the existing cluster" - type = string -} - -variable "region" { - description = "Region in which to search for the cluster" - type = string -} - -variable "additional_networks" { - description = "Additional network interface details for GKE, if any. Providing additional networks creates relevat network objects on the cluster." - default = [] - type = list(object({ - network = string - subnetwork = string - subnetwork_project = string - network_ip = string - nic_type = string - stack_type = string - queue_count = number - access_config = list(object({ - nat_ip = string - network_tier = string - })) - ipv6_access_config = list(object({ - network_tier = string - })) - alias_ip_range = list(object({ - ip_cidr_range = string - subnetwork_range_name = string - })) - })) -} - -variable "rdma_subnetwork_name_prefix" { - description = "Prefix of the RDMA subnetwork names" - default = null - type = string -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf b/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf deleted file mode 100644 index 1e458a6b4c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scheduler/pre-existing-gke-cluster/versions.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -terraform { - required_providers { - google = { - source = "hashicorp/google" - version = "> 5.0" - } - } - - provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:pre-existing-gke-cluster/v1.57.0" - } - - required_version = ">= 1.3" -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/README.md b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/README.md deleted file mode 100644 index 62407666cd..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/README.md +++ /dev/null @@ -1,348 +0,0 @@ -## Description - -This module creates a startup script that will execute a list of runners in the -order they are specified. The runners are copied to a GCS bucket at deployment -time and then copied into the VM as they are executed after startup. - -Each runner receives the following attributes: - -- `destination`: (Required) The name of the file at the destination VM. If an - absolute path is provided, the file will be copied to that path, otherwise - the file will be created in a temporary folder and deleted once the startup - script runs. -- `type`: (Required) The type of the runner, one of the following: - - `shell`: The runner is a shell script and will be executed once copied to - the destination VM. - - `ansible-local`: The runner is an ansible playbook and will run on the VM - with the following command line flags: - - ```shell - ansible-playbook --connection=local --inventory=localhost, \ - --limit localhost <> - ``` - - - `data`: The data or file specified will be copied to `<>`. No - action will be performed after the data is staged. This data can be used by - subsequent runners or simply made available on the VM for later use. -- `content`: (Optional) Content to be uploaded and, if `type` is - either `shell` or `ansible-local`, executed. Must be defined if `source` is - not. -- `source`: (Optional) A path to the file or data you want to upload. Must be - defined if `content` is not. The source path is relative to the deployment - group directory. To ensure correctness of path use `ghpc_stage` function, that - would copy referenced file to the deployment group directory. For example: - - ```yaml - source: $(ghpc_stage("path/to/file")) - ``` - - For more examples with context, see the - [example blueprint snippet](#example). To reference any other source file, an - absolute path must be used. - -- `args`: (Optional) Arguments to be passed to `shell` or `ansible-local` - runners. For `shell` runners, these will be passed as arguments to the script - when it is executed. For `ansible-local` runners, they will be appended to - a list of default arguments that invoke `ansible-playbook` on the localhost. - Therefore`args` should not include any arguments that alter this behavior, - such as `--connection`, `--inventory`, or `--limit`. - -### Runner dependencies - -`ansible-local` runners require Ansible to be installed in the VM before -running. To support other playbook runners in the Cluster Toolkit, we install -version 2.11 of `ansible-core` as well as the larger package of collections -found in `ansible` version 4.10.0. - -If an `ansible-local` runner is found in the list supplied to this module, -a script to install Ansible will be prepended to the list of runners. This -behavior can be disabled by setting `var.prepend_ansible_installer` to `false`. -This script will do the following at VM startup: - -- Install system-wide python3 if not already installed using system package - managers (yum, apt-get, etc) -- Install `python3-distutils` system-wide in debian and ubuntu based - environments. This can be a missing dependency on system installations of - python3 for installing and upgrading pip. -- Install system-wide pip3 if not already installed and upgrade pip3 if the - version is not at least 18.0. -- Install and create a virtual environment located at `/usr/local/ghpc-venv`. -- Install ansible into this virtual environment if the current version of - ansible is not version 2.11 or higher. - -To use the virtual environment created by this script, you can activate it by -running the following command on the VM: - -```shell -source /usr/local/ghpc-venv/bin/activate -``` - -You may also need to provide the correct python interpreter as the python3 -binary in the virtual environment. This can be done by adding the following flag -when calling `ansible-playbook`: - -```shell --e ansible_python_interpreter=/usr/local/ghpc-venv/bin/activate -``` - -> **_NOTE:_** ansible-playbook and other ansible command line tools will only be -> accessible from the command line (and in your PATH variable) after activating -> this environment. - -### Staging the runners - -Runners will be uploaded to a -[GCS bucket](https://cloud.google.com/storage/docs/creating-buckets). This -bucket will be created by this module and named as -`${var.deployment_name}-startup-scripts-${random_id}`. VMs using the startup -script created by this module will pull the runners content from a GCS bucket -and therefore must have access to GCS. - -> **_NOTE:_** To ensure access to GCS, set the following OAuth scope on the -> instance using the startup scripts: -> `https://www.googleapis.com/auth/devstorage.read_only`. -> -> This is set as a default scope in the [vm-instance], -> [schedMD-slurm-on-gcp-login-node] and [schedMD-slurm-on-gcp-controller] -> modules - -[vm-instance]: ../../compute/vm-instance/README.md -[schedMD-slurm-on-gcp-login-node]: ../../../community/modules/scheduler/schedmd-slurm-gcp-v6-login/README.md -[schedMD-slurm-on-gcp-controller]: ../../../community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md - -### Tracking startup script execution - -For more information on how to use startup scripts on Google Cloud Platform, -please refer to -[this document](https://cloud.google.com/compute/docs/instances/startup-scripts/linux). - -To debug startup scripts from a Linux VM created with startup script generated -by this module: - -```shell -sudo DEBUG=1 google_metadata_script_runner startup -``` - -To view outputs from a Linux startup script, run: - -```shell -sudo journalctl -u google-startup-scripts.service -``` - -### Monitoring Agent Installation - -This `startup-script` module has several options for installing a Google -monitoring agent. There are two relevant settings: `install_stackdriver_agent` -and `install_cloud_ops_agent`. - -The _Stackdriver Agent_ also called the _Legacy Cloud Monitoring Agent_ provides -better performance under some HPC workloads. While official documentation -recommends using the _Cloud Ops Agent_, it is recommended to use -`install_stackdriver_agent` when performance is important. - -#### Stackdriver Agent Installation - -If an image or machine already has Cloud Ops Agent installed and you would like -to instead use the Stackdriver Agent, the following script will remove the Cloud -Ops Agent and install the Stackdriver Agent. - -```bash -# Remove Cloud Ops Agent -sudo systemctl stop google-cloud-ops-agent.service -sudo systemctl disable google-cloud-ops-agent.service -curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh -sudo bash add-google-cloud-ops-agent-repo.sh --uninstall -sudo bash add-google-cloud-ops-agent-repo.sh --remove-repo - -# Install Stackdriver Agent -curl -sSO https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh -sudo bash add-monitoring-agent-repo.sh --also-install -curl -sSO https://dl.google.com/cloudagents/add-logging-agent-repo.sh -sudo bash add-logging-agent-repo.sh --also-install -sudo service stackdriver-agent start -``` - -#### Cloud Ops Agent Installation - -If an image or machine already has the Stackdriver Agent installed and you would -like to instead use the Cloud Ops Agent, the following script will remove the -Stackdriver Agent and install the Cloud Ops Agent. - -```bash -# UnInstall Stackdriver Agent - -sudo systemctl stop stackdriver-agent.service -sudo systemctl disable stackdriver-agent.service -curl -sSO https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh -sudo dpkg --configure -a -sudo bash add-monitoring-agent-repo.sh --uninstall -sudo bash add-monitoring-agent-repo.sh --remove-repo - -# Install ops-agent - -curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh -sudo bash add-google-cloud-ops-agent-repo.sh --also-install -sudo service google-cloud-ops-agent start -``` - -As a reminder, this should be in a startup script, which should run on all -Compute nodes via the `compute_startup_script` on the controller. - -#### Testing Installation - -You can test if one of the agents is running using the following commands: - -```bash -# For Cloud Ops Agent -$ sudo systemctl is-active google-cloud-ops-agent"*" -active -active -active -active - -# For Legacy Monitoring and Logging Agents -$ sudo service stackdriver-agent status -stackdriver-agent is running [ OK ] -$ sudo service google-fluentd status -google-fluentd is running [ OK ] -``` - -For official documentation see troubleshooting docs: - -- [Cloud Ops Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/troubleshoot-install-startup) -- [Legacy Monitoring Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/monitoring/troubleshooting) -- [Legacy Logging Agent](https://cloud.google.com/stackdriver/docs/solutions/agents/logging/troubleshooting) - -### Example - -```yaml -- id: startup - source: modules/scripts/startup-script - settings: - runners: - # Some modules such as filestore have runners as outputs for convenience: - - $(homefs.install_nfs_client_runner) - # These runners can still be created manually: - # - type: shell - # destination: "modules/filestore/scripts/install_nfs_client.sh" - # source: "modules/filestore/scripts/install_nfs_client.sh" - - type: ansible-local - destination: "modules/filestore/scripts/mount.yaml" - source: "modules/filestore/scripts/mount.yaml" - - type: data - source: /tmp/foo.tgz - destination: /tmp/bar.tgz - - type: shell - destination: "decompress.sh" - content: | - #!/bin/sh - echo $2 - tar zxvf /tmp/$1 -C / - args: "bar.tgz 'Expanding file'" - -- id: compute-cluster - source: modules/compute/vm-instance - use: [homefs, startup] -``` - -In the above example, a new GCS bucket is created to upload the startup-scripts. -But in the case where the user wants to reuse existing GCS bucket or folder, -they are able to do so by using the `gcs_bucket_path` as shown in the below example - -```yaml -- id: startup - source: modules/scripts/startup-script - settings: - gcs_bucket_path: gs://user-test-bucket/folder1/folder2 - install_stackdriver_agent: true - -- id: compute-cluster - source: modules/compute/vm-instance - use: [startup] -``` - -## License - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5 | -| [google](#requirement\_google) | >= 3.83 | -| [local](#requirement\_local) | >= 2.0.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | >= 3.83 | -| [local](#provider\_local) | >= 2.0.0 | -| [random](#provider\_random) | ~> 3.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket.configs_bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | -| [google_storage_bucket_iam_binding.viewers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_binding) | resource | -| [google_storage_bucket_object.scripts](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | -| [local_file.debug_file](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | -| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [ansible\_virtualenv\_path](#input\_ansible\_virtualenv\_path) | Virtual environment path in which to install Ansible | `string` | `"/usr/local/ghpc-venv"` | no | -| [bucket\_viewers](#input\_bucket\_viewers) | Additional service accounts or groups, users, and domains to which to grant read-only access to startup-script bucket (leave unset if using default Compute Engine service account) | `list(string)` | `[]` | no | -| [configure\_ssh\_host\_patterns](#input\_configure\_ssh\_host\_patterns) | If specified, it will automate ssh configuration by:
- Defining a Host block for every element of this variable and setting StrictHostKeyChecking to 'No'.
Ex: "hpc*", "hpc01*", "ml*"
- The first time users log-in, it will create ssh keys that are added to the authorized keys list
This requires a shared /home filesystem and relies on specifying the right prefix. | `list(string)` | `[]` | no | -| [debug\_file](#input\_debug\_file) | Path to an optional local to be written with 'startup\_script'. | `string` | `null` | no | -| [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used to name GCS bucket for startup scripts. | `string` | n/a | yes | -| [docker](#input\_docker) | Install and configure Docker | 
object({
    enabled        = optional(bool, false)
    world_writable = optional(bool, false)
    daemon_config  = optional(string, "")
  })
| 
{
  "enabled": false
}
| no | -| [enable\_docker\_world\_writable](#input\_enable\_docker\_world\_writable) | DEPRECATED: use var.docker | `bool` | `null` | no | -| [enable\_gpu\_network\_wait\_online](#input\_enable\_gpu\_network\_wait\_online) | Enable a SystemD unit that blocks execution of startup-scripts until after all network interfaces are online. (Works on reboots or boots of an image built using this solution) | `bool` | `false` | no | -| [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | `null` | no | -| [http\_no\_proxy](#input\_http\_no\_proxy) | Domains for which to disable http\_proxy behavior. Honored only if var.http\_proxy is set | `string` | `".google.com,.googleapis.com,metadata.google.internal,localhost,127.0.0.1"` | no | -| [http\_proxy](#input\_http\_proxy) | Web (http and https) proxy configuration for pip, apt, and yum/dnf and interactive shells | `string` | `""` | no | -| [install\_ansible](#input\_install\_ansible) | Run Ansible installation script if either set to true or unset and runner of type 'ansible-local' are used. | `bool` | `null` | no | -| [install\_cloud\_ops\_agent](#input\_install\_cloud\_ops\_agent) | Warning: Consider using `install_stackdriver_agent` for better performance. Run Google Ops Agent installation script if set to true. | `bool` | `false` | no | -| [install\_cloud\_rdma\_drivers](#input\_install\_cloud\_rdma\_drivers) | If true, will install and reload Cloud RDMA drivers. Currently only supported on Rocky Linux 8. Should not be enabled if using the HPC VM Image. | `bool` | `false` | no | -| [install\_docker](#input\_install\_docker) | DEPRECATED: use var.docker. | `bool` | `null` | no | -| [install\_stackdriver\_agent](#input\_install\_stackdriver\_agent) | Run Google Stackdriver Agent installation script if set to true. Preferred over ops agent for performance. | `bool` | `false` | no | -| [labels](#input\_labels) | Labels for the created GCS bucket. Key-value pairs. | `map(string)` | n/a | yes | -| [local\_ssd\_filesystem](#input\_local\_ssd\_filesystem) | Create and mount a filesystem from local SSD disks (data will be lost if VMs are powered down without enabling migration); enable by setting mountpoint field to a valid directory path. | 
object({
    fs_type     = optional(string, "ext4")
    mountpoint  = optional(string, "")
    permissions = optional(string, "0755")
  })
| 
{
  "fs_type": "ext4",
  "mountpoint": "",
  "permissions": "0755"
}
| no | -| [managed\_lustre](#input\_managed\_lustre) | Configure Managed Lustre (assumes driver already installed) | 
object({
    enabled = optional(bool, false)
    port    = optional(number, 988)
  })
| 
{
  "enabled": false,
  "port": 988
}
| no | -| [prepend\_ansible\_installer](#input\_prepend\_ansible\_installer) | DEPRECATED. Use `install_ansible=false` to prevent ansible installation. | `bool` | `null` | no | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | -| [region](#input\_region) | The region to deploy to | `string` | n/a | yes | -| [runners](#input\_runners) | List of runners to run on remote VM.
Runners can be of type ansible-local, shell or data.
A runner must specify one of 'source' or 'content'.
All runners must specify 'destination'. If 'destination' does not include a
path, it will be copied in a temporary folder and deleted after running.
Runners may also pass 'args', which will be passed as argument to shell runners only. | `list(map(string))` | `[]` | no | -| [set\_ofi\_cloud\_rdma\_tunables](#input\_set\_ofi\_cloud\_rdma\_tunables) | Controls whether to enable specific OFI environment variables for workloads using Cloud RDMA networking. Should be false for non-RDMA workloads. | `bool` | `false` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [compute\_startup\_script](#output\_compute\_startup\_script) | script to load and run all runners, as a string value. Targets the inputs for the slurm controller. | -| [controller\_startup\_script](#output\_controller\_startup\_script) | script to load and run all runners, as a string value. Targets the inputs for the slurm controller. | -| [startup\_script](#output\_startup\_script) | script to load and run all runners, as a string value. | - diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml deleted file mode 100644 index 02c449c7cb..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/configure-ssh.yml +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Configure ssh between nodes - become: true - hosts: localhost - vars: - ssh_config_path: "/etc/ssh/ssh_config" - bashrc: "{{ '/etc/bashrc' if ansible_facts['os_family'] == 'RedHat' else '/etc/bash.bashrc' }}" - setup_ssh_script: "/bin/bash /usr/local/ghpc/setup-ssh-keys.sh" - tasks: - - name: "Set StrictHostKeyChecking to no" - ansible.builtin.blockinfile: - path: "{{ ssh_config_path }}" - block: | - Host "{{ item }}" - StrictHostKeyChecking no - marker: "# {mark} ANSIBLE MANAGED BLOCK {{item}}" - loop: "{{ host_name_prefix }}" - - name: "Create ssh keys in .bashrc if not already done" - ansible.builtin.lineinfile: - path: "{{ bashrc }}" - regexp: '^{{ setup_ssh_script }}' - line: "{{ setup_ssh_script }}" diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh deleted file mode 100644 index 38c7ff9b5c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/configure_proxy.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -o pipefail - -web_proxy="${1:-}" -if [ -z "$web_proxy" ]; then - echo "Error: must provide 1 argument identifying http/https proxy" - exit 1 -fi - -# configure pip to use proxy -PIP_CONF=/etc/pip.conf -if [ ! -f "$PIP_CONF" ]; then - cat <<-EOF >"$PIP_CONF" - [global] - proxy=$web_proxy - EOF -fi - -# configure yum or dnf to use proxy -if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || - [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - YUM_CONF="/etc/yum.conf" - if ! grep -q '^proxy=.*' "$YUM_CONF"; then - sed --follow-symlinks -i.bak "/^\[main]/a proxy=$web_proxy" "$YUM_CONF" - else - sed --follow-symlinks -i.bak "s,proxy=.*,proxy=$web_proxy," "$YUM_CONF" - fi -fi - -# configure apt to use proxy -if [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release 2>/dev/null || - grep -qi ubuntu /etc/os-release 2>/dev/null; then - APT_CONF_PROXY="/etc/apt/apt.conf.d/99proxy.conf" - if [ ! -f "$APT_CONF_PROXY" ]; then - cat <<-EOF >"$APT_CONF_PROXY" - Acquire::http::Proxy "$web_proxy"; - Acquire::https::Proxy "$web_proxy"; - EOF - fi -fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh deleted file mode 100644 index 682e1352a1..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/early_run_hotfixes.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script applies fixes to VMs that must occur early in boot. For example, -# when yum or apt repositories are misconfigured, preventing most package -# operations from completing successfully. - -source /etc/os-release - -if [[ "$PRETTY_NAME" == "CentOS Linux 7 (Core)" ]]; then - echo "Applying hotfixes for CentOS 7" - if grep -q '^mirrorlist' /etc/yum.repos.d/CentOS-Base.repo; then - echo "Removing mirrorlist from default CentOS 7 repositories" - sed -i '/^mirrorlist/d' /etc/yum.repos.d/CentOS-Base.repo - fi - if grep -q '^#baseurl=http://mirror.centos.org' /etc/yum.repos.d/CentOS-Base.repo; then - echo "Reconfiguring default CentOS 7 repositories to use CentOS Vault" - sed -i 's,^#baseurl=http://mirror.centos.org/,baseurl=http://vault.centos.org/,' /etc/yum.repos.d/CentOS-Base.repo - fi -fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh deleted file mode 100644 index 3a29ae808f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/get_from_bucket.sh +++ /dev/null @@ -1,73 +0,0 @@ -#! /bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Given a url and filename, download an object to the vardir. When the installed -# version of gcloud is >=402.0.0 (Sept. 2022), then gcloud storage is used to -# fetch from the bucket. Otherwise gsutil is used. Note, the service account for -# the instance must be properly configured with a role having authorization to -# get objects from the bucket. -# -# This function is intended for single file downloads and no attempt is made to -# verify the checksum other than the default behavior of gcloud or gsutil. -# -# This function has no other platform dependencies other than gcloud / gsutil. - -# This code originated from: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 -stdlib::get_from_bucket() { - local OPTIND opt url fname dir="${VARDIR:-/var/lib/startup}" - while getopts ":u:f:d:" opt; do - case "${opt}" in - u) url="${OPTARG}" ;; - f) fname="${OPTARG}" ;; - d) dir="${OPTARG}" ;; - :) - stdlib::mandatory_argument -n stdlib::get_from_bucket -f "$OPTARG" - return "${E_MISSING_MANDATORY_ARG}" - ;; - *) - stdlib::error 'Usage: stdlib::get_from_bucket -u -f -d ' - stdlib::info 'For example: stdlib::get_from_bucket -u gs://mybucket/foo.tgz -d /var/tmp' - return "${E_UNKNOWN_ARG}" - ;; - esac - done - # Trivially compute the filename from the URL if unspecified. - if [[ -z ${fname} ]]; then - fname=${url##*/} - stdlib::debug "Computed filename='${fname}' given URL." - fi - [[ -d ${dir} ]] || mkdir "${dir}" - local attempt=0 - local max_retries=7 - # store gcs command as array and then split when called by stdlib::cmd - if stdlib::cmd gcloud help storage cp &>/dev/null; then - gcs_command=(gcloud storage cp --no-user-output-enabled) - else - gcs_command=(gsutil -q cp) - fi - while [[ $attempt -le $max_retries ]]; do - if [[ $attempt -gt 0 ]]; then - local wait=$((2 ** attempt)) - stdlib::error "Retry attempt ${attempt} of ${max_retries} with exponential backoff: ${wait} seconds." - sleep $wait - fi - if stdlib::cmd "${gcs_command[@]}" "${url}" "${dir}/${fname}"; then - break - else - stdlib::error "${gcs_command[*]} reported non-zero exit code fetching ${url}." - ((attempt++)) - fi - done -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh deleted file mode 100644 index eac2b2e32a..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_ansible.sh +++ /dev/null @@ -1,247 +0,0 @@ -#!/bin/sh -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ex -REQ_ANSIBLE_VERSION=2.15 -REQ_ANSIBLE_PIP_VERSION=8.7.0 -REQ_PIP_WHEEL_VERSION=0.45.1 -REQ_PIP_SETUPTOOLS_VERSION=80.8.0 -REQ_PIP_MAJOR_VERSION=25 -REQ_PYTHON3_VERSION=9 - -apt_wait() { - while fuser /var/lib/apt/lists/lock >/dev/null 2>&1; do - echo "Sleeping for apt lists lock" - sleep 3 - done -} - -# Installs any dependencies needed for python based on the OS -install_python_deps() { - # this file is present on both Debian and Ubuntu OSes - if [ -f /etc/debian_version ]; then - apt_wait - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get install -o DPkg::Lock::Timeout=600 -y python3-setuptools python3-venv - fi -} - -# Gets the name of the python executable for python starting with python3, then -# checking python. Sets the variable to an empty string if neither are found. -get_python_path() { - python_path="" - if command -v python3 1>/dev/null; then - python_path=$(command -v python3) - elif command -v python 1>/dev/null; then - python_path=$(command -v python) - fi -} - -# Returns the python major version. If provided, it will use the first argument -# as the python executable, otherwise it will default to simply "python". -get_python_major_version() { - python_path=${1:-python} - python_major_version=$(${python_path} -c "import sys; print(sys.version_info.major)") -} - -# Returns the python minor version. If provided, it will use the first argument -# as the python executable, otherwise it will default to simply "python". -get_python_minor_version() { - python_path=${1:-python} - python_minor_version=$(${python_path} -c "import sys; print(sys.version_info.minor)") -} - -# Install python3 with the yum package manager. Updates python_path to the -# newly installed packaged. -install_python3_dnf() { - major_version=$(rpm -E "%{rhel}") - set -- "--disablerepo=*" "--enablerepo=baseos,appstream" - if grep -qi 'ID="rhel"' /etc/os-release; then - # Do not set --disablerepo / --enablerepo on RedHat, due to - # complex repo names; clear array - set -- - fi - # On Rocky Linux 9, Python 3.9 is installed by default but this - # has already been dropped by ansible-core for control nodes. - # https://docs.ansible.com/ansible/latest/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix - # Python 3.12 aligns with RHEL 10 default (GA: 13 May 2025) where - # it is available as "python3*" but must be named explicitly on - # older releases. It also ensures longer support for Ansible. - if [ "${major_version}" -lt "10" ]; then - dnf install "$@" -y python3.12 python3.12-pip - python_path=$(command -v python3.12) - else - dnf install "$@" -y python3 python3-pip - python_path=$(command -v python3) - fi -} - -# Install python3 with the apt package manager. Updates python_path to the -# newly installed packaged. -install_python3_apt() { - apt_wait - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get install -o DPkg::Lock::Timeout=600 -y python3 python3-setuptools python3-pip python3-venv - python_path=$(command -v python3) -} - -install_python3() { - if [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || - [ -f /etc/system-release ]; then - install_python3_dnf - elif [ -f /etc/debian_version ]; then - install_python3_apt - else - echo "Error: Unsupported Distribution" - return 1 - fi -} - -# Install pip3 with the dnf package manager. Updates python_path to the -# newly installed packaged. -install_pip3_dnf() { - major_version=$(rpm -E "%{rhel}") - set -- "--disablerepo=*" "--enablerepo=baseos,appstream" - if grep -qi 'ID="rhel"' /etc/os-release; then - # Do not set --disablerepo / --enablerepo on RedHat, due to complex repo names - # clear array - set -- - fi - # Python 3.12 aligns with RHEL 10 default (GA: 13 May 2025) where - # it is available as "python3*" but must be named explicitly on - # older releases. It also ensures longer support for Ansible. - if [ "${major_version}" -lt "10" ]; then - dnf install "$@" -y python3.12-pip - else - dnf install "$@" -y python3-pip - fi -} - -# Install pip3 with the apt package manager. Updates python_path to the -# newly installed packaged. -install_pip3_apt() { - apt_wait - apt-get update --allow-releaseinfo-change-origin --allow-releaseinfo-change-label - apt-get install -o DPkg::Lock::Timeout=600 -y python3-pip -} - -install_pip3() { - if [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || - [ -f /etc/system-release ]; then - install_pip3_dnf - elif [ -f /etc/debian_version ]; then - install_pip3_apt - else - echo "Error: Unsupported Distribution" - return 1 - fi -} - -main() { - if [ $# -gt 1 ]; then - echo "Error: provide only 1 optional argument identifying virtual environment path for Ansible" - return 1 - fi - - venv_path="${1:-/usr/local/ghpc-venv}" - - # Get the python3 executable, or install it if not found - get_python_path - get_python_major_version "${python_path}" - get_python_minor_version "${python_path}" - if [ "${python_path}" = "" ] || [ "${python_major_version}" = "2" ] || [ "${python_minor_version}" -lt "${REQ_PYTHON3_VERSION}" ]; then - if ! install_python3; then - return 1 - fi - get_python_major_version "${python_path}" - get_python_minor_version "${python_path}" - else - install_python_deps - fi - - # Install OS-packaged pip - if ! ${python_path} -m pip --version 2>/dev/null; then - if ! install_pip3; then - return 1 - fi - fi - - # Create pip virtual environment for Cluster Toolkit - ${python_path} -m venv "${venv_path}" --copies - venv_python_path=${venv_path}/bin/python3 - - # Upgrade pip if necessary - pip_version=$(${venv_python_path} -m pip --version | sed -nr 's/^pip ([0-9]+\.[0-9]+).*$/\1/p') - pip_major_version=$(echo "${pip_version}" | cut -d '.' -f 1) - if [ "${pip_major_version}" -lt "${REQ_PIP_MAJOR_VERSION}" ]; then - ${venv_python_path} -m pip install --upgrade pip - fi - - # upgrade wheel if necessary - wheel_pkg=$(${venv_python_path} -m pip list --format=freeze | grep "^wheel" || true) - if [ "$wheel_pkg" != "wheel==${REQ_PIP_WHEEL_VERSION}" ]; then - ${venv_python_path} -m pip install -U wheel==${REQ_PIP_WHEEL_VERSION} - fi - - # upgrade setuptools if necessary - setuptools_pkg=$(${venv_python_path} -m pip list --format=freeze | grep "^setuptools" || true) - if [ "$setuptools_pkg" != "setuptools==${REQ_PIP_SETUPTOOLS_VERSION}" ]; then - ${venv_python_path} -m pip install -U setuptools==${REQ_PIP_SETUPTOOLS_VERSION} - fi - - # configure ansible to always use correct Python binary - if [ ! -f /etc/ansible/ansible.cfg ]; then - mkdir /etc/ansible - cat <<-EOF >/etc/ansible/ansible.cfg - [defaults] - interpreter_python=${venv_python_path} - stdout_callback=debug - stderr_callback=debug - EOF - fi - - # Install ansible - ansible_version="" - if command -v ansible-playbook 1>/dev/null; then - ansible_version=$(ansible-playbook --version 2>/dev/null | sed -nr 's/^ansible-playbook.*([0-9]+\.[0-9]+\.[0-9]+).*/\1/p') - ansible_major_vers=$(echo "${ansible_version}" | cut -d '.' -f 1) - ansible_minor_vers=$(echo "${ansible_version}" | cut -d '.' -f 2) - ansible_req_major_vers=$(echo "${REQ_ANSIBLE_VERSION}" | cut -d '.' -f 1) - ansible_req_minor_vers=$(echo "${REQ_ANSIBLE_VERSION}" | cut -d '.' -f 2) - fi - if [ -z "${ansible_version}" ] || [ "${ansible_major_vers}" -ne "${ansible_req_major_vers}" ] || - [ "${ansible_minor_vers}" -lt "${ansible_req_minor_vers}" ]; then - ${venv_python_path} -m pip install ansible=="${REQ_ANSIBLE_PIP_VERSION}" - fi - while read -r cmd; do - if ! [ -L "/usr/bin/${cmd}" ]; then - ln -s "${venv_path}/bin/${cmd}" "/usr/bin/${cmd}" - fi - done <<-EOF - ansible - ansible-config - ansible-connection - ansible-console - ansible-doc - ansible-galaxy - ansible-inventory - ansible-playbook - ansible-pull - ansible-test - ansible-vault - EOF -} - -main "$@" diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh deleted file mode 100644 index 375792459b..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_cloud_rdma_drivers.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -o pipefail - -OS_ID="$(awk -F '=' '/^ID=/ {print $2}' /etc/os-release | sed -e 's/"//g')" -OS_VERSION="$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g')" -OS_VERSION_MAJOR="$(awk -F '=' '/VERSION_ID/ {print $2}' /etc/os-release | sed -e 's/"//g' -e 's/\..*$//')" -REBOOT_FILE="/etc/.rdma_reboot" - -if { [ "${OS_ID}" = "rocky" ] || [ "${OS_ID}" = "rhel" ]; } && { [ "${OS_VERSION_MAJOR}" = "8" ]; }; then - KMOD_VERSION="$(dnf list installed | awk '$1 ~ /^kmod-idpf-irdma(\.|$)/ {print $2}')" - - # For images that do not already have Cloud RDMA drivers installed - if [ -z "${KMOD_VERSION}" ] && [ -z "${REBOOT_FILE}" ]; then - sudo dnf update -y - sudo dnf install https://depot.ciq.com/public/files/gce-accelerator/irdma-kernel-modules-el8-x86_64/irdma-repos.rpm -y - sudo dnf install kmod-idpf-irdma rdma-core libibverbs-utils librdmacm-utils infiniband-diags perftest -y - sudo touch "${REBOOT_FILE}" - reboot - fi - echo "This image has IRDMA packages already installed, exiting." - exit 0 -else - echo "Unsupported operating system ${OS_ID} ${OS_VERSION}. Cloud RDMA Drivers are only supported on Rocky Linux 8." - exit 1 -fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_docker.yml b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_docker.yml deleted file mode 100644 index af251014b8..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_docker.yml +++ /dev/null @@ -1,167 +0,0 @@ -# Copyright 2024 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Install and configure Docker - hosts: all - become: true - vars: - docker_data_root: '' - docker_daemon_config: '' - enable_docker_world_writable: false - is_rocky_linux: "{{ ansible_distribution in ['Rocky', 'Rocky Linux'] }}" - tasks: - - name: Gather distribution facts - ansible.builtin.setup: - gather_subset: platform - - name: Check if docker is installed - ansible.builtin.stat: - path: /usr/bin/docker - register: docker_binary - - name: Add Docker GPG key (Rocky) - ansible.builtin.rpm_key: - state: present - key: https://download.docker.com/linux/rhel/gpg - when: - - is_rocky_linux - - not docker_binary.stat.exists - register: add_docker_key - retries: 30 - delay: 10 - until: add_docker_key is succeeded - - name: Add Docker repository for Rocky Linux - ansible.builtin.command: > - dnf config-manager --add-repo - https://download.docker.com/linux/rhel/docker-ce.repo - when: - - is_rocky_linux - - not docker_binary.stat.exists - notify: Refresh dnf cache - - name: Install Docker Engine and related packages on Rocky - ansible.builtin.package: - name: - - docker-ce - - docker-ce-cli - - containerd.io - - docker-buildx-plugin - - docker-compose-plugin - state: present - update_cache: yes - when: - - is_rocky_linux - - not docker_binary.stat.exists - - name: Ensure Docker SDK for Python is installed - block: - - name: Install python3-pip - ansible.builtin.package: - name: python3-pip - state: present - - name: Install Docker SDK for Python - ansible.builtin.pip: - name: docker - executable: /usr/local/ghpc-venv/bin/pip3 - when: is_rocky_linux - - name: Download Docker Installer (get.docker.com) - ansible.builtin.get_url: - url: https://get.docker.com - dest: /tmp/get-docker.sh - owner: root - group: root - mode: '0644' - when: - - not is_rocky_linux - - not docker_binary.stat.exists - - name: Install Docker - ansible.builtin.command: sh /tmp/get-docker.sh - register: docker_installed - changed_when: docker_installed.rc != 0 - when: - - not is_rocky_linux - - not docker_binary.stat.exists - - name: Create Docker daemon configuration - ansible.builtin.copy: - dest: /etc/docker/daemon.json - mode: '0644' - content: '{{ docker_daemon_config }}' - validate: /usr/bin/dockerd --validate --config-file %s - when: docker_daemon_config - notify: - - Restart Docker - - name: Create Docker service override directory - ansible.builtin.file: - path: /etc/systemd/system/docker.service.d - state: directory - owner: root - group: root - mode: '0755' - - name: Create Docker service override configuration - ansible.builtin.copy: - dest: /etc/systemd/system/docker.service.d/data-root.conf - mode: '0644' - content: | - [Unit] - {% if docker_data_root %} - RequiresMountsFor={{ docker_data_root }} - {% endif %} - After=mount-localssd-raid.service - - name: Create Docker socket override directory - ansible.builtin.file: - path: /etc/systemd/system/docker.socket.d - state: directory - owner: root - group: root - mode: '0755' - when: enable_docker_world_writable - - name: Create Docker socket override configuration - ansible.builtin.copy: - dest: /etc/systemd/system/docker.socket.d/world-writable.conf - mode: '0644' - content: | - [Socket] - SocketMode=0666 - when: enable_docker_world_writable - notify: - - Reload SystemD - - Recreate Docker socket - - name: Delete Docker socket override configuration - ansible.builtin.file: - path: /etc/systemd/system/docker.socket.d/world-writable.conf - state: absent - when: not enable_docker_world_writable - notify: - - Reload SystemD - - Recreate Docker socket - - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true - - name: Recreate Docker socket - ansible.builtin.service: - name: docker.socket - state: restarted - - name: Restart Docker - ansible.builtin.service: - name: docker.service - state: restarted - - name: Refresh dnf cache - ansible.builtin.dnf: - update_cache: yes - - post_tasks: - - name: Start Docker - ansible.builtin.service: - name: docker.service - state: started - enabled: true diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml deleted file mode 100644 index 7f3dda3d24..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_gpu_network_wait_online.yml +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Wait until all network interfaces are online - hosts: all - become: true - tasks: - - name: Create SystemD service for A3 High networking - when: ansible_os_family == "Debian" - ansible.builtin.copy: - dest: /etc/systemd/system/delay-a3-high.service - owner: root - group: root - mode: "0644" - content: | - [Unit] - Description=Delay A3 High boot until all network interfaces are routable - After=network-online.target - Wants=network-online.target - Before=google-startup-scripts.service - - [Service] - ExecCondition=/bin/bash -c '/usr/bin/curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetata/v1/instance/machine-type | grep -q "/a3-highgpu-8g$"' - ExecStart=/usr/lib/systemd/systemd-networkd-wait-online -i enp0s12 -i enp6s0 -i enp12s0 -i enp134s0 -i enp140s0 -o routable --timeout=180 - ExecStartPost=/bin/sleep 30 - - [Install] - WantedBy=multi-user.target - notify: - - Reload SystemD - - Enable A3 High delay - handlers: - - name: Reload SystemD - ansible.builtin.systemd: - daemon_reload: true - post_tasks: - - name: Enable A3 High delay - # by the time this startup-script executes, we don't care if it's started - # just enabled - ansible.builtin.systemd_service: - name: delay-a3-high - enabled: true diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml deleted file mode 100644 index 94699471bb..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_managed_lustre.yml +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2025 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- name: Configure Managed Lustre (assumes driver already installed) - hosts: all - become: true - vars: - default_lustre_port: 988 - managed_lustre_port: "{{ default_lustre_port }}" - tasks: - # Ideally changes to this file would also trigger an execution of lnetctl - # command to update accept_port but it is unclear if lnetctl supports this. - - name: Update lnet to use non-default port - when: managed_lustre_port | int != {{ default_lustre_port }} - ansible.builtin.copy: - owner: root - group: root - mode: '0644' - dest: /etc/modprobe.d/lnet.conf - content: | - options lnet accept_port={{ managed_lustre_port | int }} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh deleted file mode 100644 index eb4bf899b8..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/install_monitoring_agent.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e -o pipefail - -LEGACY_MONITORING_PACKAGE='stackdriver-agent' -LEGACY_MONITORING_SCRIPT_URL='https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh' -LEGACY_LOGGING_PACKAGE='google-fluentd' -LEGACY_LOGGING_SCRIPT_URL='https://dl.google.com/cloudagents/add-logging-agent-repo.sh' - -OPSAGENT_PACKAGE='google-cloud-ops-agent' -OPSAGENT_SCRIPT_URL='https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh' - -ops_or_legacy="${1:-legacy}" - -fail() { - echo >&2 "[$(date +'%Y-%m-%dT%H:%M:%S%z')] $*" - exit 1 -} - -handle_debian() { - is_legacy_monitoring_installed() { - dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${LEGACY_MONITORING_PACKAGE} | - grep "${LEGACY_MONITORING_PACKAGE} is installed" - } - - is_legacy_logging_installed() { - dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${LEGACY_LOGGING_PACKAGE} | - grep "${LEGACY_LOGGING_PACKAGE} is installed" - } - - is_legacy_installed() { - is_legacy_monitoring_installed || is_legacy_logging_installed - } - - is_opsagent_installed() { - dpkg-query --show --showformat 'dpkg-query: ${Package} is installed\n' ${OPSAGENT_PACKAGE} | - grep "${OPSAGENT_PACKAGE} is installed" - } - - install_with_retry() { - MAX_RETRY=50 - RETRY=0 - until [ ${RETRY} -eq ${MAX_RETRY} ] || curl -s "${1}" | bash -s -- --also-install; do - RETRY=$((RETRY + 1)) - echo "WARNING: Installation of ${1} failed on try ${RETRY} of ${MAX_RETRY}" - sleep 5 - done - if [ $RETRY -eq $MAX_RETRY ]; then - echo "ERROR: Installation of ${1} was not successful after ${MAX_RETRY} attempts." - exit 1 - fi - } - - install_opsagent() { - install_with_retry "${OPSAGENT_SCRIPT_URL}" - } - - install_stackdriver_agent() { - install_with_retry "${LEGACY_MONITORING_SCRIPT_URL}" - install_with_retry "${LEGACY_LOGGING_SCRIPT_URL}" - service stackdriver-agent start - service google-fluentd start - } -} - -handle_redhat() { - is_legacy_monitoring_installed() { - rpm --query --queryformat 'package %{NAME} is installed\n' ${LEGACY_MONITORING_PACKAGE} | - grep "${LEGACY_MONITORING_PACKAGE} is installed" - } - - is_legacy_logging_installed() { - rpm --query --queryformat 'package %{NAME} is installed\n' ${LEGACY_LOGGING_PACKAGE} | - grep "${LEGACY_LOGGING_PACKAGE} is installed" - } - - is_legacy_installed() { - is_legacy_monitoring_installed || is_legacy_logging_installed - } - - is_opsagent_installed() { - rpm --query --queryformat 'package %{NAME} is installed\n' ${OPSAGENT_PACKAGE} | - grep "${OPSAGENT_PACKAGE} is installed" - } - - install_opsagent() { - curl -s "${OPSAGENT_SCRIPT_URL}" | bash -s -- --also-install - } - - install_stackdriver_agent() { - curl -sS "${LEGACY_MONITORING_SCRIPT_URL}" | bash -s -- --also-install - curl -sS "${LEGACY_LOGGING_SCRIPT_URL}" | bash -s -- --also-install - service stackdriver-agent start - service google-fluentd start - } -} - -main() { - if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || [ -f /etc/oracle-release ] || [ -f /etc/system-release ]; then - handle_redhat - elif [ -f /etc/debian_version ] || grep -qi ubuntu /etc/lsb-release || grep -qi ubuntu /etc/os-release; then - handle_debian - else - fail "Unsupported platform." - fi - - # Handle cases that agent is already installed - if [[ -z "$(is_legacy_monitoring_installed)" && -n $(is_legacy_logging_installed) ]] || - [[ -n "$(is_legacy_monitoring_installed)" && -z $(is_legacy_logging_installed) ]]; then - fail "Bad state: legacy agent is partially installed" - elif [[ "${ops_or_legacy}" == "legacy" ]] && is_legacy_installed; then - echo "Legacy agent is already installed" - exit 0 - elif [[ "${ops_or_legacy}" != "legacy" ]] && is_opsagent_installed; then - echo "Ops agent is already installed" - exit 0 - elif is_legacy_installed || is_opsagent_installed; then - fail "Agent is already installed but does not match requested agent of ${ops_or_legacy}" - fi - - # install agent - if [[ "${ops_or_legacy}" == "legacy" ]]; then - echo "Installing legacy monitoring agent (stackdriver)" - install_stackdriver_agent - else - echo "Installing cloud ops agent" - echo "WARNING: cloud ops agent may have a performance impact. Consider using legacy monitoring agent (stackdriver)." - install_opsagent - fi -} - -main diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh deleted file mode 100644 index 738181aafb..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/running-script-warning.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -SCRIPT_COMPLETE_FILE="/run/startup_script_msg" - -# Ensure we're in an interactive terminal and not root -if [ -t 1 ] && [ "$(id -u)" -ne 0 ]; then - # Check if the file has contents otherwise skip - if [ -s "$SCRIPT_COMPLETE_FILE" ]; then - echo - cat "$SCRIPT_COMPLETE_FILE" - echo - fi -fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml deleted file mode 100644 index d94aac81fd..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-raid.yml +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Configure local SSDs - become: true - hosts: localhost - vars: - raid_name: localssd - array_dev: /dev/md/{{ raid_name }} - fstype: ext4 - interface: nvme - mode: '0755' - mountpoint: /mnt/{{ raid_name }} - tasks: - - name: Get local SSD devices - ansible.builtin.find: - file_type: link - path: /dev/disk/by-id - patterns: google-local-{{ "nvme-" if interface == "nvme" else "" }}ssd-* - register: local_ssd_devices - - - name: Exit if zero local ssd found - ansible.builtin.meta: end_play - when: local_ssd_devices.files | length == 0 - - - name: Install mdadm - ansible.builtin.package: - name: mdadm - state: present - - # this service will act during the play and upon reboots to ensure that local - # SSD volumes are always assembled into a RAID and re-formatted if necessary; - # there are many scenarios where a VM can be stopped or migrated during - # maintenance and the contents of local SSD will be discarded - - name: Install service to create local SSD RAID and format it - ansible.builtin.copy: - dest: /etc/systemd/system/create-localssd-raid.service - mode: 0644 - content: | - [Unit] - After=local-fs.target - Before=slurmd.service docker.service - ConditionPathExists=!{{ array_dev }} - - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStart=/usr/bin/bash -c "/usr/sbin/mdadm --create {{ array_dev }} --name={{ raid_name }} --homehost=any --level=0 --raid-devices={{ local_ssd_devices.files | length }} /dev/disk/by-id/google-local-nvme-ssd-*{{ " --force" if local_ssd_devices.files | length == 1 else "" }}" - ExecStartPost=/usr/sbin/mkfs -t {{ fstype }}{{ " -m 0" if fstype == "ext4" else "" }} {{ array_dev }} - - [Install] - WantedBy=slurmd.service docker.service - - - name: Create RAID array and format - ansible.builtin.systemd: - name: create-localssd-raid.service - state: started - enabled: true - daemon_reload: true - - - name: Install service to mount local SSD array - ansible.builtin.copy: - dest: /etc/systemd/system/mount-localssd-raid.service - mode: 0644 - content: | - [Unit] - After=local-fs.target create-localssd-raid.service - Before=slurmd.service docker.service - Wants=create-localssd-raid.service - ConditionPathIsMountPoint=!{{ mountpoint }} - - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStart=/usr/bin/systemd-mount -t {{ fstype }} -o discard,defaults,nofail {{ array_dev }} {{ mountpoint }} - ExecStartPost=/usr/bin/chmod {{ mode }} {{ mountpoint }} - ExecStop=/usr/bin/systemd-umount {{ mountpoint }} - - [Install] - WantedBy=slurmd.service docker.service - - - name: Mount RAID array and set permissions - ansible.builtin.systemd: - name: mount-localssd-raid.service - state: started - enabled: true - daemon_reload: true diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh deleted file mode 100644 index 1c8018fb01..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ! -d ~/.ssh/ ]; then - source /usr/local/ghpc-venv/bin/activate - ansible-playbook /usr/local/ghpc/setup-ssh-keys.yml -fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml deleted file mode 100644 index 692896bb9c..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/setup-ssh-keys.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Setup SSH Keys for user - become: false - hosts: localhost - vars: - pub_key_path: "{{ ansible_env.HOME }}/.ssh" - pub_key_file: "{{ pub_key_path }}/id_rsa" - auth_key_file: "{{ pub_key_path }}/authorized_keys" - tasks: - - name: "Create .ssh folder" - ansible.builtin.file: - path: "{{ pub_key_path }}" - state: directory - mode: 0700 - owner: "{{ ansible_user_id }}" - - name: Create keys - community.crypto.openssh_keypair: - path: "{{ pub_key_file }}" - owner: "{{ ansible_user_id }}" - - name: Copy public key to authorized keys - ansible.builtin.copy: - src: "{{ pub_key_file }}.pub" - dest: "{{ auth_key_file }}" - owner: "{{ ansible_user_id }}" - mode: 0644 diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh deleted file mode 100644 index 8ca40bc73f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-body.sh +++ /dev/null @@ -1,39 +0,0 @@ -#! /bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This code contains minor changes from the original: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 - -stdlib::main() { - DELETE_AT_EXIT="$(mktemp -d)" - readonly DELETE_AT_EXIT - - # Initialize state required by other functions, e.g. debug() - stdlib::init - stdlib::debug "Loaded startup-script-stdlib as an executable." - - stdlib::load_config_values - - stdlib::load_runners -} - -# if script is being executed and not sourced. -if [[ ${BASH_SOURCE[0]} == "${0}" ]]; then - stdlib::finish() { - [[ -d ${DELETE_AT_EXIT:-} ]] && rm -rf "${DELETE_AT_EXIT}" - } - trap stdlib::finish EXIT - - stdlib::main "$@" -fi diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh deleted file mode 100644 index 589a3215ab..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/files/startup-script-stdlib-head.sh +++ /dev/null @@ -1,266 +0,0 @@ -#! /bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This code contains minor changes from the original in: https://github.com/terraform-google-modules/terraform-google-startup-scripts?ref=v1.0.0 - -# Standard library of functions useful for startup scripts. - -# These are outside init_global_vars so logging functions work with the most -# basic case of `source startup-script-stdlib.sh` -readonly SYSLOG_DEBUG_PRIORITY="${SYSLOG_DEBUG_PRIORITY:-syslog.debug}" -readonly SYSLOG_INFO_PRIORITY="${SYSLOG_INFO_PRIORITY:-syslog.info}" -readonly SYSLOG_ERROR_PRIORITY="${SYSLOG_ERROR_PRIORITY:-syslog.error}" -# Global counter of how many times stdlib::init() has been called. -STARTUP_SCRIPT_STDLIB_INITIALIZED=0 - -# Error codes -readonly E_RUN_OR_DIE=5 -readonly E_MISSING_MANDATORY_ARG=9 -readonly E_UNKNOWN_ARG=10 - -SCRIPT_COMPLETE_FILE="/run/startup_script_msg" -SUCCESS_MESSAGE="* NOTICE **: The Cluster Toolkit startup scripts have finished running successfully." -readonly SUCCESS_MESSAGE -ERROR_MESSAGE="** ERROR **: The Cluster Toolkit startup scripts have finished running, but produced an error." -readonly ERROR_MESSAGE -WARNING_MESSAGE="** WARNING **: The Cluster Toolkit startup scripts are currently running." -readonly WARNING_MESSAGE - -stdlib::debug() { - [[ -z ${DEBUG:-} ]] && return 0 - local ds msg - msg="$*" - logger -p "${SYSLOG_DEBUG_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" - [[ -n ${QUIET:-} ]] && return 0 - ds="$(date +"${DATE_FMT}") " - echo -e "${BLUE}${ds}Debug [$$]: ${msg}${NC}" >&2 -} - -stdlib::info() { - local ds msg - msg="$*" - logger -p "${SYSLOG_INFO_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" - [[ -n ${QUIET:-} ]] && return 0 - ds="$(date +"${DATE_FMT}") " - echo -e "${GREEN}${ds}Info [$$]: ${msg}${NC}" >&2 -} - -stdlib::error() { - local ds msg - msg="$*" - ds="$(date +"${DATE_FMT}") " - logger -p "${SYSLOG_ERROR_PRIORITY}" -t "${PROG}[$$]" -- "${msg}" - echo -e "${RED}${ds}Error [$$]: ${msg}${NC}" >&2 -} - -stdlib::announce_runners_start() { - if [ -z "$recursive_proc" ]; then - wall -n "$WARNING_MESSAGE" - echo "$WARNING_MESSAGE" >"$SCRIPT_COMPLETE_FILE" - fi - export recursive_proc=$((${recursive_proc:=0} + 1)) -} - -stdlib::announce_runners_end() { - exit_code=$1 - export recursive_proc=$((${recursive_proc:=0} - 1)) - if [ "$recursive_proc" -le "0" ]; then - if [ "$exit_code" -ne "0" ]; then - wall -n "$ERROR_MESSAGE" - echo "$ERROR_MESSAGE" >"$SCRIPT_COMPLETE_FILE" - else - wall -n "$SUCCESS_MESSAGE" - echo -n "" >"$SCRIPT_COMPLETE_FILE" - fi - fi -} - -# The main initialization function of this library. This should be kept to the -# minimum amount of work required for all functions to operate cleanly. -stdlib::init() { - if [[ ${STARTUP_SCRIPT_STDLIB_INITIALIZED} -gt 0 ]]; then - stdlib::info 'stdlib::init()'" already initialized, no action taken." - return 0 - fi - ((STARTUP_SCRIPT_STDLIB_INITIALIZED++)) || true - stdlib::init_global_vars - stdlib::init_directories - stdlib::debug "stdlib::init(): startup-script-stdlib.sh initialized and ready" -} - -# Initialize global variables. -stdlib::init_global_vars() { - # The program name, used for logging. - readonly PROG="${PROG:-startup-script-stdlib}" - # Date format used for stderr logging. Passed to date + command. - readonly DATE_FMT="${DATE_FMT:-"%a %b %d %H:%M:%S %z %Y"}" - # var directory - readonly VARDIR="${VARDIR:-/var/lib/startup}" - # Override this with file://localhost/tmp/foo/bar in spec test context - readonly METADATA_BASE="${METADATA_BASE:-http://metadata.google.internal}" - - # Color variables - if [[ -n ${COLOR:-} ]]; then - readonly NC='\033[0m' # no color - readonly RED='\033[0;31m' # error - readonly GREEN='\033[0;32m' # info - readonly BLUE='\033[0;34m' # debug - else - readonly NC='' - readonly RED='' - readonly GREEN='' - readonly BLUE='' - fi - - return 0 -} - -stdlib::init_directories() { - if ! [[ -e ${VARDIR} ]]; then - install -d -m 0755 -o 0 -g 0 "${VARDIR}" - fi -} - -## -# Get a metadata key. When used without -o, this function is guaranteed to -# produce no output on STDOUT other than the retrieved value. This is intended -# to support the use case of -# FOO="$(stdlib::metadata_get -k instance/attributes/foo)" -# -# If the requested key does not exist, the error code will be 22 and zero bytes -# written to STDOUT. -stdlib::metadata_get() { - local OPTIND opt key outfile - local metadata="${METADATA_BASE%/}/computeMetadata/v1" - local exit_code - while getopts ":k:o:" opt; do - case "${opt}" in - k) key="${OPTARG}" ;; - o) outfile="${OPTARG}" ;; - :) - stdlib::error "Invalid option: -${OPTARG} requires an argument" - stdlib::metadata_get_usage - return "${E_MISSING_MANDATORY_ARG}" - ;; - *) - stdlib::error "Unknown option: -${opt}" - stdlib::metadata_get_usage - return "${E_UNKNOWN_ARG}" - ;; - esac - done - local url="${metadata}/${key#/}" - - stdlib::debug "Getting metadata resource url=${url}" - if [[ -z ${outfile:-} ]]; then - curl --location --silent --connect-timeout 1 --fail \ - -H 'Metadata-Flavor: Google' "$url" 2>/dev/null - exit_code=$? - else - stdlib::cmd curl --location \ - --silent \ - --connect-timeout 1 \ - --fail \ - --output "${outfile}" \ - -H 'Metadata-Flavor: Google' \ - "$url" - exit_code=$? - fi - case "${exit_code}" in - 22 | 37) - stdlib::debug "curl exit_code=${exit_code} for url=${url}" \ - "(Does not exist)" - ;; - esac - return "${exit_code}" -} - -stdlib::metadata_get_usage() { - stdlib::info 'Usage: stdlib::metadata_get -k ' - stdlib::info 'For example: stdlib::metadata_get -k instance/attributes/startup-config' -} - -# Load configuration values in the spirit of /etc/sysconfig defaults, but from -# metadata instead of the filesystem. -stdlib::load_config_values() { - local config_file - local key="instance/attributes/startup-script-config" - # shellcheck disable=SC2119 - config_file="$(stdlib::mktemp)" - stdlib::metadata_get -k "${key}" -o "${config_file}" - local status=$? - case "$status" in - 0) - stdlib::debug "SUCCESS: Configuration data sourced from $key" - ;; - 22 | 37) - stdlib::debug "no configuration data loaded from $key" - ;; - *) - stdlib::error "metadata_get -k $key returned unknown status=${status}" - ;; - esac - # shellcheck source=/dev/null - source "${config_file}" -} - -# Run a command logging the entry and exit. Intended for system level commands -# and operational debugging. Not intended for use with redirection. This is -# not named run() because bats uses a run() function. -stdlib::cmd() { - local exit_code argv=("$@") - stdlib::debug "BEGIN: stdlib::cmd() command=[${argv[*]}]" - "${argv[@]}" - exit_code=$? - stdlib::debug "END: stdlib::cmd() command=[${argv[*]}] exit_code=${exit_code}" - return $exit_code -} - -# Run a command successfully or exit the program with an error. -stdlib::run_or_die() { - if ! stdlib::cmd "$@"; then - stdlib::error "stdlib::run_or_die(): exiting with exit code ${E_RUN_OR_DIE}." - exit "${E_RUN_OR_DIE}" - fi -} - -# Intended to take advantage of automatic cleanup of startup script library -# temporary files without exporting a modified TMPDIR to child processes, which -# would cause the children to have their TMPDIR deleted out from under them. -# shellcheck disable=SC2120 -stdlib::mktemp() { - TMPDIR="${DELETE_AT_EXIT:-${TMPDIR}}" mktemp "$@" -} - -# Return a nice error message if a mandatory argument is missing. -stdlib::mandatory_argument() { - local OPTIND opt name flag - while getopts ":n:f:" opt; do - case "$opt" in - n) name="${OPTARG}" ;; - f) flag="${OPTARG}" ;; - :) - stdlib::error "Invalid argument: -${OPTARG} requires an argument to stdlib::mandatory_argument()" - return "${E_MISSING_MANDATORY_ARG}" - ;; - *) - stdlib::error "Unknown argument: -${OPTARG}" - stdlib::info "Usage: stdlib::mandatory_argument -n -f " - return "${E_UNKNOWN_ARG}" - ;; - esac - done - stdlib::error "Invalid argument: -${flag} requires an argument to ${name}()." -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/main.tf b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/main.tf deleted file mode 100644 index d91fb3edb1..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/main.tf +++ /dev/null @@ -1,305 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - # This label allows for billing report tracking based on module. - labels = merge(var.labels, { ghpc_module = "startup-script", ghpc_role = "scripts" }) -} - -locals { - monitoring_agent_installer = ( - var.install_cloud_ops_agent || var.install_stackdriver_agent ? - [{ - type = "shell" - source = "${path.module}/files/install_monitoring_agent.sh" - destination = "install_monitoring_agent_automatic.sh" - args = var.install_cloud_ops_agent ? "ops" : "legacy" # install legacy (stackdriver) - }] : - [] - ) - - warnings = [ - { - type = "data" - content = file("${path.module}/files/running-script-warning.sh") - destination = "/etc/profile.d/99-running-script-warning.sh" - } - ] - - configure_ssh = length(var.configure_ssh_host_patterns) > 0 - host_args = { - host_name_prefix = var.configure_ssh_host_patterns - } - - prefix_file = "/tmp/prefix_file.json" - ansible_docker_settings_file = "/tmp/ansible_docker_settings.json" - - docker_config = try(jsondecode(var.docker.daemon_config), {}) - docker_data_root = try(local.docker_config.data-root, null) - - configure_ssh_runners = local.configure_ssh ? [ - { - type = "data" - source = "${path.module}/files/setup-ssh-keys.sh" - destination = "/usr/local/ghpc/setup-ssh-keys.sh" - }, - { - type = "data" - source = "${path.module}/files/setup-ssh-keys.yml" - destination = "/usr/local/ghpc/setup-ssh-keys.yml" - }, - { - type = "data" - content = jsonencode(local.host_args) - destination = local.prefix_file - }, - { - type = "ansible-local" - content = file("${path.module}/files/configure-ssh.yml") - destination = "configure-ssh.yml" - args = "-e @${local.prefix_file}" - } - ] : [] - - proxy_runner = var.http_proxy == "" ? [] : [ - { - type = "data" - destination = "/etc/profile.d/http_proxy.sh" - content = <<-EOT - #!/bin/bash - export http_proxy=${var.http_proxy} - export https_proxy=${var.http_proxy} - export NO_PROXY=${var.http_no_proxy} - EOT - }, - { - type = "shell" - source = "${path.module}/files/configure_proxy.sh" - destination = "configure_proxy.sh" - args = var.http_proxy - } - ] - - ofi_runner = !var.set_ofi_cloud_rdma_tunables ? [] : [ - { - type = "data" - destination = "/etc/profile.d/set_ofi_cloud_rdma_tunables.sh" - content = <<-EOT - #!/bin/bash - export FI_PROVIDER="verbs;ofi_rxm" - export FI_OFI_RXM_USE_RNDV_WRITE=0 - export FI_VERBS_INLINE_SIZE=39 - export I_MPI_FABRICS="shm:ofi" - export FI_UNIVERSE_SIZE=3072 - export I_MPI_ADJUST_ALLTOALL=1 - export I_MPI_ADJUST_IALLTOALL=1 - export I_MPI_ADJUST_BCAST=4 - export I_MPI_ADJUST_IBCAST=1 - EOT - }, - ] - - rdma_runner = !var.install_cloud_rdma_drivers ? [] : [ - { - type = "shell" - source = "${path.module}/files/install_cloud_rdma_drivers.sh" - destination = "install_cloud_rdma_drivers.sh" - } - ] - - docker_runner = !var.docker.enabled ? [] : [ - { - type = "data" - destination = local.ansible_docker_settings_file - content = jsonencode({ - enable_docker_world_writable = var.docker.world_writable - docker_daemon_config = var.docker.daemon_config - docker_data_root = local.docker_data_root - }) - }, - { - type = "ansible-local" - destination = "install_docker.yml" - content = file("${path.module}/files/install_docker.yml") - args = "-e \"@${local.ansible_docker_settings_file}\"" - }, - ] - - managed_lustre_runner = !var.managed_lustre.enabled ? [] : [ - { - type = "ansible-local" - destination = "install_managed_lustre.yml" - content = file("${path.module}/files/install_managed_lustre.yml") - args = "-e managed_lustre_port=${var.managed_lustre.port}" - }, - ] - - gpu_network_wait_online_runner = !var.enable_gpu_network_wait_online ? [] : [ - { - type = "ansible-local" - destination = "install_gpu_network_wait_online.yml" - content = file("${path.module}/files/install_gpu_network_wait_online.yml") - args = "" - }, - ] - - local_ssd_filesystem_enabled = can(coalesce(var.local_ssd_filesystem.mountpoint)) - raid_setup = !local.local_ssd_filesystem_enabled ? [] : [ - { - type = "ansible-local" - destination = "setup-raid.yml" - content = file("${path.module}/files/setup-raid.yml") - args = join(" ", [ - "-e mountpoint=${var.local_ssd_filesystem.mountpoint}", - "-e fs_type=${var.local_ssd_filesystem.fs_type}", - "-e mode=${var.local_ssd_filesystem.permissions}", - ]) - }, - ] - - supplied_ansible_runners = anytrue([for r in var.runners : r.type == "ansible-local"]) - has_ansible_runners = anytrue([ - local.supplied_ansible_runners, - local.configure_ssh, - var.docker.enabled, - var.managed_lustre.enabled, - var.enable_gpu_network_wait_online, - local.local_ssd_filesystem_enabled - ]) - - install_ansible = coalesce(var.install_ansible, local.has_ansible_runners) - ansible_installer = local.install_ansible ? [{ - type = "shell" - source = "${path.module}/files/install_ansible.sh" - destination = "install_ansible_automatic.sh" - args = var.ansible_virtualenv_path - }] : [] - - hotfix_runner = [{ - type = "shell" - source = "${path.module}/files/early_run_hotfixes.sh" - destination = "early_run_hotfixes.sh" - }] - - runners = concat( - local.warnings, - local.hotfix_runner, - local.proxy_runner, - local.ofi_runner, - local.rdma_runner, - local.monitoring_agent_installer, - local.ansible_installer, - local.raid_setup, # order RAID early to ensure filesystem is ready for subsequent runners - local.managed_lustre_runner, - local.configure_ssh_runners, - local.docker_runner, - local.gpu_network_wait_online_runner, - var.runners - ) - - bucket_regex = "^gs://([^/]*)/*(.*)" - gcs_bucket_path_trimmed = var.gcs_bucket_path == null ? null : trimsuffix(var.gcs_bucket_path, "/") - storage_folder_path = local.gcs_bucket_path_trimmed == null ? null : regex(local.bucket_regex, local.gcs_bucket_path_trimmed)[1] - storage_folder_path_prefix = local.storage_folder_path == null || local.storage_folder_path == "" ? "" : "${local.storage_folder_path}/" - - user_provided_bucket_name = try(regex(local.bucket_regex, local.gcs_bucket_path_trimmed)[0], null) - storage_bucket_name = coalesce(one(google_storage_bucket.configs_bucket[*].name), local.user_provided_bucket_name) - - load_runners = templatefile( - "${path.module}/templates/startup-script-custom.tftpl", - { - bucket = local.storage_bucket_name, - http_proxy = var.http_proxy, - no_proxy = var.http_no_proxy, - runners = [ - for runner in local.runners : { - object = google_storage_bucket_object.scripts[basename(runner["destination"])].output_name - type = runner["type"] - destination = runner["destination"] - args = contains(keys(runner), "args") ? runner["args"] : "" - } - ] - } - ) - - stdlib_head = file("${path.module}/files/startup-script-stdlib-head.sh") - get_from_bucket = file("${path.module}/files/get_from_bucket.sh") - stdlib_body = file("${path.module}/files/startup-script-stdlib-body.sh") - - # List representing complete content, to be concatenated together. - stdlib_list = [ - local.stdlib_head, - local.get_from_bucket, - local.load_runners, - local.stdlib_body, - ] - - # Final content output to the user - stdlib = join("", local.stdlib_list) - - runners_map = { for runner in local.runners : - basename(runner["destination"]) => { - content = lookup(runner, "content", null) - source = lookup(runner, "source", null) - } - } -} - -resource "random_id" "resource_name_suffix" { - byte_length = 4 -} - -resource "google_storage_bucket" "configs_bucket" { - count = var.gcs_bucket_path == null ? 1 : 0 - project = var.project_id - name = "${var.deployment_name}-startup-scripts-${random_id.resource_name_suffix.hex}" - uniform_bucket_level_access = true - location = var.region - storage_class = "REGIONAL" - labels = local.labels -} - -resource "google_storage_bucket_iam_binding" "viewers" { - bucket = local.storage_bucket_name - role = "roles/storage.objectViewer" - members = var.bucket_viewers -} - -resource "google_storage_bucket_object" "scripts" { - # this writes all scripts exactly once into GCS - for_each = local.runners_map - name = "${local.storage_folder_path_prefix}${each.key}-${substr(try(md5(each.value.content), filemd5(each.value.source)), 0, 4)}" - content = each.value.content - source = each.value.source - bucket = local.storage_bucket_name - timeouts { - create = "10m" - update = "10m" - } - - lifecycle { - precondition { - condition = !(var.install_cloud_ops_agent && var.install_stackdriver_agent) - error_message = "Only one of var.install_stackdriver_agent or var.install_cloud_ops_agent can be set. Stackdriver is recommended for best performance." - } - } -} - -resource "local_file" "debug_file" { - for_each = toset(var.debug_file != null ? [var.debug_file] : []) - filename = var.debug_file - content = local.stdlib -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/metadata.yaml b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/metadata.yaml deleted file mode 100644 index 2ada34471f..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/metadata.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 "Google LLC" -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -spec: - requirements: - services: - - storage.googleapis.com diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/outputs.tf b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/outputs.tf deleted file mode 100644 index 6a15082814..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/outputs.tf +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "startup_script" { - description = "script to load and run all runners, as a string value." - value = local.stdlib - depends_on = [ - google_storage_bucket_iam_binding.viewers - ] -} - -output "compute_startup_script" { - description = "script to load and run all runners, as a string value. Targets the inputs for the slurm controller." - value = local.stdlib - depends_on = [ - google_storage_bucket_iam_binding.viewers - ] -} - -output "controller_startup_script" { - description = "script to load and run all runners, as a string value. Targets the inputs for the slurm controller." - value = local.stdlib - depends_on = [ - google_storage_bucket_iam_binding.viewers - ] -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl deleted file mode 100644 index 3c894b00b0..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/templates/startup-script-custom.tftpl +++ /dev/null @@ -1,65 +0,0 @@ - - -stdlib::run_playbook() { - if [ ! "$(which ansible-playbook)" ]; then - stdlib::error "ansible-playbook not found"\ - "Please install ansible before running ansible-local runners." - exit 1 - fi - ansible-playbook --connection=local --inventory=localhost, --limit localhost $1 $2 - ret_code=$? - return $${ret_code} -} - -stdlib::runner() { - - type=$1 - object=$2 - destination=$3 - tmpdir=$4 - args=$5 - - destpath="$(dirname $destination)" - filename="$(basename $destination)" - - if [ "$destpath" = "." ]; then - destpath=$tmpdir - fi - - stdlib::get_from_bucket -u "gs://${bucket}/$object" -d "$destpath" -f "$filename" - - stdlib::info "=== start executing runner: $object ===" - case "$1" in - ansible-local) stdlib::run_playbook "$destpath/$filename" "$args";; - shell) chmod u+x /$destpath/$filename && $destpath/$filename $args;; - esac - - exit_code=$? - stdlib::info "=== $object finished with exit_code=$exit_code ===" - if [ "$exit_code" -ne "0" ] ; then - stdlib::error "=== execution of $object failed, exiting ===" - stdlib::announce_runners_end "$exit_code" - exit $exit_code - fi -} - -stdlib::load_runners(){ - tmpdir="$(mktemp -d)" - - stdlib::debug "=== BEGIN Running runners ===" - stdlib::announce_runners_start - - %{if http_proxy != "" ~} - stdlib::info "=== Setting HTTP_PROXY,HTTPS_PROXY to ${http_proxy} ===" - export http_proxy=${http_proxy} - export https_proxy=${http_proxy} - export NO_PROXY=${no_proxy} - %{endif ~} - - %{for r in runners ~} - stdlib::runner "${r.type}" "${r.object}" "${r.destination}" $${tmpdir} "${r.args}" - %{endfor ~} - - stdlib::announce_runners_end "0" - stdlib::debug "=== END Running runners ===" -} diff --git a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/variables.tf b/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/variables.tf deleted file mode 100644 index 7080085ece..0000000000 --- a/vdi-test-scott/primary/modules/embedded/modules/scripts/startup-script/variables.tf +++ /dev/null @@ -1,298 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "Project in which the HPC deployment will be created" - type = string -} - -variable "deployment_name" { - description = "Name of the HPC deployment, used to name GCS bucket for startup scripts." - type = string -} - -variable "region" { - description = "The region to deploy to" - type = string -} - -variable "gcs_bucket_path" { - description = "The GCS path for storage bucket and the object, starting with `gs://`." - type = string - default = null -} - -variable "bucket_viewers" { - description = "Additional service accounts or groups, users, and domains to which to grant read-only access to startup-script bucket (leave unset if using default Compute Engine service account)" - type = list(string) - default = [] - - validation { - condition = alltrue([ - for u in var.bucket_viewers : length(regexall("^(allUsers$|allAuthenticatedUsers$|user:|group:|serviceAccount:|domain:)", u)) > 0 - ]) - error_message = "Bucket viewer members must begin with user/group/serviceAccount/domain following https://cloud.google.com/iam/docs/reference/rest/v1/Policy#Binding" - } -} - -variable "debug_file" { - description = "Path to an optional local to be written with 'startup_script'." - type = string - default = null -} - -variable "labels" { - description = "Labels for the created GCS bucket. Key-value pairs." - type = map(string) -} - -variable "runners" { - description = < 0 - error_message = "The POSIX permissions for the mountpoint must be represented as a 3 or 4-digit octal" - } - - default = { - fs_type = "ext4" - mountpoint = "" - permissions = "0755" - } - - nullable = false -} - -variable "install_cloud_ops_agent" { - description = "Warning: Consider using `install_stackdriver_agent` for better performance. Run Google Ops Agent installation script if set to true." - type = bool - default = false -} - -variable "install_stackdriver_agent" { - description = "Run Google Stackdriver Agent installation script if set to true. Preferred over ops agent for performance." - type = bool - default = false -} - -variable "install_ansible" { - description = "Run Ansible installation script if either set to true or unset and runner of type 'ansible-local' are used." - type = bool - default = null -} - -variable "configure_ssh_host_patterns" { - description = < Date: Fri, 18 Jul 2025 11:00:24 +0100 Subject: [PATCH 13/21] precommit --- .../modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml index 96d0370f7f..52c2dd3aaf 100644 --- a/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml @@ -53,7 +53,7 @@ - name: Debug kernel GCC version extraction ansible.builtin.debug: msg: - - "Raw kernel GCC version: {{ kernel_gcc_version.stdout }}" + - "Raw kernel GCC version: {{ kernel_gcc_version.stdout }}" - "Extracted required GCC version: {{ required_gcc_version }}" - name: Get current GCC version From 437ffc571978f15c1a364da3ab4dd40be3d67531 Mon Sep 17 00:00:00 2001 From: Scott Gordon Date: Fri, 18 Jul 2025 12:09:26 +0100 Subject: [PATCH 14/21] vdi webapp port change --- .../roles/vdi_tool/tasks/guacamole.yaml | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml index 8aba005db8..e023dafe9a 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml @@ -124,7 +124,6 @@ - name: Start Guacamole PostgreSQL container ansible.builtin.command: > docker run -d --name guac_db --network guac_net - -p 5432:5432 -e POSTGRES_USER=guacamole_db -e POSTGRES_PASSWORD={{ database_password }} -e POSTGRES_DB=guacamole_db @@ -137,16 +136,25 @@ - postgres_start.rc != 0 - "'already in use' not in postgres_start.stderr" -- name: Wait for PostgreSQL on host port +- name: Wait for PostgreSQL container to be ready ansible.builtin.wait_for: host: 127.0.0.1 port: 5432 delay: 2 timeout: 60 + ignore_errors: true + +- name: Wait for PostgreSQL via Docker exec + ansible.builtin.command: docker exec guac_db pg_isready -U guacamole_db + register: postgres_ready + until: postgres_ready.rc == 0 + retries: 30 + delay: 2 - name: Start guacd container ansible.builtin.command: > - docker run -d --name guacd --network host + docker run -d --name guacd --network guac_net + -p 4822:4822 --restart always guacamole/guacd:latest register: guacd_start @@ -156,14 +164,15 @@ - name: Start Guacamole webapp container ansible.builtin.command: > - docker run -d --name guac_app --network host - -e POSTGRES_HOSTNAME=127.0.0.1 + docker run -d --name guac_app --network guac_net + -p {{ vdi_webapp_port }}:8080 + -e POSTGRES_HOSTNAME=guac_db -e POSTGRES_PORT=5432 -e POSTGRES_DATABASE=guacamole_db -e POSTGRES_USER=guacamole_db -e POSTGRES_PASSWORD={{ database_password }} -e POSTGRES_AUTO_CREATE_ACCOUNTS=true - -e GUACD_HOSTNAME=localhost + -e GUACD_HOSTNAME=guacd -e GUACD_PORT=4822 --restart always guacamole/guacamole:latest @@ -172,6 +181,15 @@ - guac_app_start.rc != 0 - "'already in use' not in guac_app_start.stderr" +- name: Debug container status before waiting for Guacamole + ansible.builtin.command: docker ps + register: container_status + changed_when: false + +- name: Show container status + ansible.builtin.debug: + msg: "{{ container_status.stdout }}" + - name: Wait for Guacamole HTTP endpoint ansible.builtin.wait_for: host: 127.0.0.1 From e90fd0a56893e55fa5df8a0abc65aff958e554fd Mon Sep 17 00:00:00 2001 From: Scott Gordon Date: Fri, 18 Jul 2025 15:02:50 +0100 Subject: [PATCH 15/21] precommit trailing spaces --- .../modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml index 52c2dd3aaf..4f1a2ec070 100644 --- a/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/ubuntu.yaml @@ -52,7 +52,7 @@ - name: Debug kernel GCC version extraction ansible.builtin.debug: - msg: + msg: - "Raw kernel GCC version: {{ kernel_gcc_version.stdout }}" - "Extracted required GCC version: {{ required_gcc_version }}" From 4d2fb3e33e63e8427cf7be47857c3ee172224c3a Mon Sep 17 00:00:00 2001 From: Scott Gordon Date: Thu, 24 Jul 2025 16:38:42 +0100 Subject: [PATCH 16/21] OFE: vdi module - improvements --- community/modules/scripts/vdi-setup/README.md | 1 + .../VDI_IDEMPOTENCY_IMPLEMENTATION_PLAN.md | 175 + community/modules/scripts/vdi-setup/logs.txt | 14395 ++++++++++++++++ community/modules/scripts/vdi-setup/main.tf | 15 +- .../vdi-setup/roles/base_os/tasks/main.yaml | 62 +- .../roles/lock_manager/tasks/check_lock.yaml | 203 + .../lock_manager/tasks/cleanup_lock.yaml | 29 + .../roles/lock_manager/tasks/create_lock.yaml | 308 + .../roles/lock_manager/tasks/main.yaml | 60 + .../lock_manager/tasks/monitor_setup.yaml | 73 + .../lock_manager/templates/lock_file.yaml.j2 | 35 + .../templates/vdi-monitor.service.j2 | 17 + .../lock_manager/templates/vdi-monitor.sh.j2 | 184 + .../roles/secret_manager/tasks/main.yaml | 203 +- .../tasks/user_secret_tasks.yaml | 59 +- .../user_provision/tasks/local_users.yaml | 52 +- .../roles/user_provision/tasks/main.yaml | 89 +- .../roles/vdi_tool/tasks/guacamole.yaml | 91 +- .../vdi-setup/roles/vdi_tool/tasks/main.yaml | 46 +- .../templates/connection_bootstrap.sql.j2 | 72 - .../guacamole/connection_bootstrap.sql.j2 | 78 + .../templates/guacamole/user_bootstrap.sql.j2 | 29 + .../templates/guacamole/user_cleanup.sql.j2 | 143 + .../vdi_tool/templates/user_bootstrap.sql.j2 | 22 - .../vdi-setup/roles/vnc/tasks/main.yaml | 42 +- .../vdi-setup/templates/install.yaml.tftpl | 9 +- .../vdi-setup/templates/vars.yaml.tftpl | 2 + .../modules/scripts/vdi-setup/variables.tf | 11 + 28 files changed, 16278 insertions(+), 227 deletions(-) create mode 100644 community/modules/scripts/vdi-setup/VDI_IDEMPOTENCY_IMPLEMENTATION_PLAN.md create mode 100644 community/modules/scripts/vdi-setup/logs.txt create mode 100644 community/modules/scripts/vdi-setup/roles/lock_manager/tasks/check_lock.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/lock_manager/tasks/cleanup_lock.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/lock_manager/tasks/create_lock.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/lock_manager/tasks/main.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/lock_manager/tasks/monitor_setup.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 create mode 100644 community/modules/scripts/vdi-setup/roles/lock_manager/templates/vdi-monitor.service.j2 create mode 100644 community/modules/scripts/vdi-setup/roles/lock_manager/templates/vdi-monitor.sh.j2 delete mode 100644 community/modules/scripts/vdi-setup/roles/vdi_tool/templates/connection_bootstrap.sql.j2 create mode 100644 community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_bootstrap.sql.j2 create mode 100644 community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/user_bootstrap.sql.j2 create mode 100644 community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/user_cleanup.sql.j2 delete mode 100644 community/modules/scripts/vdi-setup/roles/vdi_tool/templates/user_bootstrap.sql.j2 diff --git a/community/modules/scripts/vdi-setup/README.md b/community/modules/scripts/vdi-setup/README.md index 1a3e4cf7e6..3cf4feacf0 100644 --- a/community/modules/scripts/vdi-setup/README.md +++ b/community/modules/scripts/vdi-setup/README.md @@ -174,6 +174,7 @@ After deployment, you can access the VDI in several ways: | [vnc\_flavor](#input\_vnc\_flavor) | The VNC server flavor to use (tigervnc currently supported) | `string` | `"tigervnc"` | no | | [vnc\_port\_max](#input\_vnc\_port\_max) | Maximum valid VNC port. | `number` | `5999` | no | | [vnc\_port\_min](#input\_vnc\_port\_min) | Minimum valid VNC port. | `number` | `5901` | no | +| [debug](#input\_debug) | Enable debug mode for verbose logging during VDI setup. | `bool` | `false` | no | ## Outputs diff --git a/community/modules/scripts/vdi-setup/VDI_IDEMPOTENCY_IMPLEMENTATION_PLAN.md b/community/modules/scripts/vdi-setup/VDI_IDEMPOTENCY_IMPLEMENTATION_PLAN.md new file mode 100644 index 0000000000..4f1faf9eed --- /dev/null +++ b/community/modules/scripts/vdi-setup/VDI_IDEMPOTENCY_IMPLEMENTATION_PLAN.md @@ -0,0 +1,175 @@ +# VDI Idempotency Implementation Plan + +## Overview +This document outlines the implementation of idempotency for the VDI setup module, ensuring that roles can be safely re-run without causing conflicts or duplicate operations. + +## Key Features + +### 1. Lock File System +- **Location**: `/opt/vdi-setup/.vdi-lock.yaml` +- **Purpose**: Tracks deployment status, role completion, and configuration changes +- **Structure**: YAML format with deployment metadata and role status + +### 2. Role Execution Control +- **Pattern**: Each role checks if it should run before executing tasks +- **Logic**: Roles run if: + - Lock file doesn't exist (fresh deployment) + - Role is not marked as completed + - Deployment configuration has changed + - Force rerun is enabled + +### 3. Setup Status Tracking +- **Status Values**: `configuring`, `available`, `error` +- **Purpose**: Indicates current VDI setup state for external monitoring +- **Usage**: Django app can show "Offline" if VDI is offline (separate logic) + +### 4. VM Metadata Integration +- **Content**: Base64-encoded lock file stored in VM metadata +- **Purpose**: External access to VDI status without SSH access +- **Command**: `gcloud compute instances describe --format="value(metadata.items[vdi-lock-content])" | base64 -d` + +### 5. Monitoring Service +- **Service**: `vdi-monitor.service` (systemd) +- **Script**: `/opt/vdi-setup/vdi-monitor.sh` +- **Purpose**: Monitors lock file changes and triggers reconfiguration +- **Bucket Sync**: Pulls latest files from GCS bucket before reconfiguring + +## Implementation Details + +### Lock File Structure +```yaml +vdi_setup_status: + deployment_name: "vdi-test-scott" + deployment_hash: "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab" + lock_version: "1.0" + created_at: "2025-07-24T10:13:31Z" + last_updated: "2025-07-24T10:13:31Z" + force_rerun: false + setup_status: "configuring" # configuring, available, error + + completed_roles: + base_os: + completed: false + lock_manager: + completed: true + completed_at: "2025-07-24T10:13:31Z" + secret_manager: + completed: false + user_provision: + completed: false + vnc: + completed: false + vdi_tool: + completed: false + + user_secrets_status: + last_secret_check: "2025-07-24T10:13:31Z" + user_secrets_hash: "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" + users_updated: + - alice + - bob + + user_management: + current_users: + - alice + - bob + previous_users: + - alice + - bob + removed_users: [] +``` + +### Role Execution Pattern +Each role follows this pattern: +```yaml +# Check if this role should run +- name: Check if role should run + ansible.builtin.import_role: + name: lock_manager + tasks_from: check_lock + vars: + current_role: "" + +# Skip all tasks if role should not run +- name: Skip tasks if role should not run + ansible.builtin.debug: + msg: "Skipping role - already completed or not needed" + when: not role_should_run + +# Role-specific tasks with conditional execution +- name: Role task + ansible.builtin.task: + # task details + when: role_should_run + +# Mark role as completed +- name: Mark role as completed + ansible.builtin.import_role: + name: lock_manager + tasks_from: create_lock + vars: + current_role: "" + role_completed: true + when: role_should_run +``` + +### Change Detection +- **Deployment Hash**: SHA-256 hash of deployment configuration +- **User Secrets Hash**: SHA-256 hash of user configuration (blueprint data only) +- **Role Completion**: Individual tracking of each role's completion status + +### Secret Manager Integration +- **Blueprint-based Detection**: User secrets hash is calculated from blueprint data (usernames, ports, secret_names, etc.) +- **Password Changes**: Changes to passwords in Secret Manager are **not automatically detected** +- **Manual Re-initialization**: To apply password changes, use `force_rerun: true` in lock file +- **Secret ID Changes**: Changes to `secret_name` references are automatically detected + +### Monitoring Service +- **Trigger**: Lock file modification detected +- **Action**: Sync files from GCS bucket and run Ansible reconfiguration +- **Logging**: Detailed logs in `/var/log/vdi-monitor.log` + +## Benefits + +1. **Idempotency**: Safe to re-run deployment without conflicts +2. **Efficiency**: Skip completed roles to reduce deployment time +3. **Monitoring**: External visibility into VDI status +4. **Automation**: Automatic reconfiguration on configuration changes +5. **Reliability**: Proper error handling and status tracking + +## Usage + +### Manual Status Check +```bash +# Check on-disk lock file +cat /opt/vdi-setup/.vdi-lock.yaml + +# Check VM metadata +gcloud compute instances describe --zone= --format="value(metadata.items[vdi-lock-content])" | base64 -d +``` + +### Force Re-run +```bash +# Edit lock file to force re-run +sed -i 's/force_rerun: false/force_rerun: true/' /opt/vdi-setup/.vdi-lock.yaml +``` + +### Handle Secret Manager Password Changes +```bash +# To apply password changes from Secret Manager: +# 1. Update the password in Secret Manager +# 2. Force re-run to apply changes +sed -i 's/force_rerun: false/force_rerun: true/' /opt/vdi-setup/.vdi-lock.yaml + +# 3. Run the playbook again +cd /tmp/vdi && ansible-playbook install.yaml --extra-vars @vars.yaml +``` + +### Monitor Service +```bash +# Check service status +systemctl status vdi-monitor + +# View logs +tail -f /var/log/vdi-monitor.log +``` \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/logs.txt b/community/modules/scripts/vdi-setup/logs.txt new file mode 100644 index 0000000000..f5c0e0a2f5 --- /dev/null +++ b/community/modules/scripts/vdi-setup/logs.txt @@ -0,0 +1,14395 @@ +[root@vdi-test-scott-rocky-0 ~]# cat /var/log/messages | awk -F'startup-script: ' '/startup-script:/ {print $2}' +77 files removed ++ rm -rf /var/cache/yum ++ shred --remove /etc/ssh/ssh_host_ecdsa_key /etc/ssh/ssh_host_ecdsa_key.pub /etc/ssh/ssh_host_ed25519_key /etc/ssh/ssh_host_ed25519_key.pub /etc/ssh/ssh_host_rsa_key /etc/ssh/ssh_host_rsa_key.pub ++ truncate -s 0 /var/log/messages ++ rm -rf /root/mpi-tuning/ ++ rm -f /root/.bash_history ++ history -c ++ echo 'DaisySuccess: HPC VM tuning complete' +DaisySuccess: HPC VM tuning complete +Thu Jul 24 15:23:27 +0000 2025 Info [1535]: === start executing runner: 99-running-script-warning.sh-54ee === +Thu Jul 24 15:23:27 +0000 2025 Info [1535]: === 99-running-script-warning.sh-54ee finished with exit_code=0 === +Thu Jul 24 15:23:29 +0000 2025 Info [1535]: === start executing runner: early_run_hotfixes.sh-b383 === +Thu Jul 24 15:23:29 +0000 2025 Info [1535]: === early_run_hotfixes.sh-b383 finished with exit_code=0 === +Thu Jul 24 15:23:30 +0000 2025 Info [1535]: === start executing runner: passed_startup_script.sh-22f5 === +Thu Jul 24 15:23:32 +0000 2025 Info [1870]: === start executing runner: 99-running-script-warning.sh-54ee === +Thu Jul 24 15:23:32 +0000 2025 Info [1870]: === 99-running-script-warning.sh-54ee finished with exit_code=0 === +Thu Jul 24 15:23:34 +0000 2025 Info [1870]: === start executing runner: early_run_hotfixes.sh-b383 === +Thu Jul 24 15:23:34 +0000 2025 Info [1870]: === early_run_hotfixes.sh-b383 finished with exit_code=0 === +Thu Jul 24 15:23:36 +0000 2025 Info [1870]: === start executing runner: install_ansible_automatic.sh-ccf1 === ++ REQ_ANSIBLE_VERSION=2.15 ++ REQ_ANSIBLE_PIP_VERSION=8.7.0 ++ REQ_PIP_WHEEL_VERSION=0.45.1 ++ REQ_PIP_SETUPTOOLS_VERSION=80.8.0 ++ REQ_PIP_MAJOR_VERSION=25 ++ REQ_PYTHON3_VERSION=9 ++ main /usr/local/ghpc-venv ++ '[' 1 -gt 1 ']' ++ venv_path=/usr/local/ghpc-venv ++ get_python_path ++ python_path= ++ command -v python3 +++ command -v python3 ++ python_path=/usr/bin/python3 ++ get_python_major_version /usr/bin/python3 ++ python_path=/usr/bin/python3 +++ /usr/bin/python3 -c 'import sys; print(sys.version_info.major)' ++ python_major_version=3 ++ get_python_minor_version /usr/bin/python3 ++ python_path=/usr/bin/python3 +++ /usr/bin/python3 -c 'import sys; print(sys.version_info.minor)' ++ python_minor_version=6 ++ '[' /usr/bin/python3 = '' ']' ++ '[' 3 = 2 ']' ++ '[' 6 -lt 9 ']' ++ install_python3 ++ '[' -f /etc/redhat-release ']' ++ install_python3_dnf +++ rpm -E '%{rhel}' ++ major_version=8 ++ set -- '--disablerepo=*' --enablerepo=baseos,appstream ++ grep -qi 'ID="rhel"' /etc/os-release ++ '[' 8 -lt 10 ']' ++ dnf install '--disablerepo=*' --enablerepo=baseos,appstream -y python3.12 python3.12-pip +Waiting for process with pid 1124 to finish. +Last metadata expiration check: 0:00:46 ago on Thu 24 Jul 2025 03:24:31 PM UTC. +Dependencies resolved. +================================================================================ + Package Arch Version Repository Size +================================================================================ +Installing: + python3.12 x86_64 3.12.11-1.el8_10 appstream 30 k + python3.12-pip noarch 23.2.1-4.el8 appstream 3.2 M +Installing dependencies: + python3.12-libs x86_64 3.12.11-1.el8_10 appstream 10 M + python3.12-pip-wheel noarch 23.2.1-4.el8 appstream 1.5 M +Installing weak dependencies: + python3.12-setuptools noarch 68.2.2-5.el8_10 appstream 1.7 M + +Transaction Summary +================================================================================ +Install 5 Packages + +Total download size: 16 M +Installed size: 65 M +Downloading Packages: +(1/5): python3.12-3.12.11-1.el8_10.x86_64.rpm 858 kB/s | 30 kB 00:00 +(2/5): python3.12-libs-3.12.11-1.el8_10.x86_64. 73 MB/s | 10 MB 00:00 +(3/5): python3.12-pip-wheel-23.2.1-4.el8.noarch 11 MB/s | 1.5 MB 00:00 +(4/5): python3.12-setuptools-68.2.2-5.el8_10.no 6.1 MB/s | 1.7 MB 00:00 +(5/5): python3.12-pip-23.2.1-4.el8.noarch.rpm 6.0 MB/s | 3.2 MB 00:00 +-------------------------------------------------------------------------------- +Total 27 MB/s | 16 MB 00:00 +Running transaction check +Transaction check succeeded. +Running transaction test +Transaction test succeeded. +Running transaction + Preparing : 1/1 + Installing : python3.12-pip-wheel-23.2.1-4.el8.noarch 1/5 + Installing : python3.12-libs-3.12.11-1.el8_10.x86_64 2/5 + Installing : python3.12-3.12.11-1.el8_10.x86_64 3/5 + Running scriptlet: python3.12-3.12.11-1.el8_10.x86_64 3/5 + Installing : python3.12-setuptools-68.2.2-5.el8_10.noarch 4/5 + Installing : python3.12-pip-23.2.1-4.el8.noarch 5/5 + Running scriptlet: python3.12-pip-23.2.1-4.el8.noarch 5/5 + Verifying : python3.12-3.12.11-1.el8_10.x86_64 1/5 + Verifying : python3.12-libs-3.12.11-1.el8_10.x86_64 2/5 + Verifying : python3.12-pip-23.2.1-4.el8.noarch 3/5 + Verifying : python3.12-pip-wheel-23.2.1-4.el8.noarch 4/5 + Verifying : python3.12-setuptools-68.2.2-5.el8_10.noarch 5/5 + +Installed: + python3.12-3.12.11-1.el8_10.x86_64 + python3.12-libs-3.12.11-1.el8_10.x86_64 + python3.12-pip-23.2.1-4.el8.noarch + python3.12-pip-wheel-23.2.1-4.el8.noarch + python3.12-setuptools-68.2.2-5.el8_10.noarch + +Complete! +++ command -v python3.12 ++ python_path=/usr/bin/python3.12 ++ get_python_major_version /usr/bin/python3.12 ++ python_path=/usr/bin/python3.12 +++ /usr/bin/python3.12 -c 'import sys; print(sys.version_info.major)' ++ python_major_version=3 ++ get_python_minor_version /usr/bin/python3.12 ++ python_path=/usr/bin/python3.12 +++ /usr/bin/python3.12 -c 'import sys; print(sys.version_info.minor)' ++ python_minor_version=12 ++ /usr/bin/python3.12 -m pip --version +pip 23.2.1 from /usr/lib/python3.12/site-packages/pip (python 3.12) ++ /usr/bin/python3.12 -m venv /usr/local/ghpc-venv --copies ++ venv_python_path=/usr/local/ghpc-venv/bin/python3 +++ /usr/local/ghpc-venv/bin/python3 -m pip --version +++ sed -nr 's/^pip ([0-9]+\.[0-9]+).*$/\1/p' ++ pip_version=23.2 +++ echo 23.2 +++ cut -d . -f 1 ++ pip_major_version=23 ++ '[' 23 -lt 25 ']' ++ /usr/local/ghpc-venv/bin/python3 -m pip install --upgrade pip +Requirement already satisfied: pip in /usr/local/ghpc-venv/lib64/python3.12/site-packages (23.2.1) +Collecting pip + Obtaining dependency information for pip from https://files.pythonhosted.org/packages/29/a2/d40fb2460e883eca5199c62cfc2463fd261f760556ae6290f88488c362c0/pip-25.1.1-py3-none-any.whl.metadata + Downloading pip-25.1.1-py3-none-any.whl.metadata (3.6 kB) +Downloading pip-25.1.1-py3-none-any.whl (1.8 MB) + ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā” 1.8/1.8 MB 28.2 MB/s eta 0:00:00 +Installing collected packages: pip + Attempting uninstall: pip + Found existing installation: pip 23.2.1 + Uninstalling pip-23.2.1: + Successfully uninstalled pip-23.2.1 +Successfully installed pip-25.1.1 +++ /usr/local/ghpc-venv/bin/python3 -m pip list --format=freeze +++ grep '^wheel' +++ true ++ wheel_pkg= ++ '[' '' '!=' wheel==0.45.1 ']' ++ /usr/local/ghpc-venv/bin/python3 -m pip install -U wheel==0.45.1 +Collecting wheel==0.45.1 + Downloading wheel-0.45.1-py3-none-any.whl.metadata (2.3 kB) +Downloading wheel-0.45.1-py3-none-any.whl (72 kB) +Installing collected packages: wheel +Successfully installed wheel-0.45.1 +++ /usr/local/ghpc-venv/bin/python3 -m pip list --format=freeze +++ grep '^setuptools' +++ true ++ setuptools_pkg= ++ '[' '' '!=' setuptools==80.8.0 ']' ++ /usr/local/ghpc-venv/bin/python3 -m pip install -U setuptools==80.8.0 +Collecting setuptools==80.8.0 + Downloading setuptools-80.8.0-py3-none-any.whl.metadata (6.6 kB) +Downloading setuptools-80.8.0-py3-none-any.whl (1.2 MB) + ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā” 1.2/1.2 MB 29.3 MB/s eta 0:00:00 +Installing collected packages: setuptools +Successfully installed setuptools-80.8.0 ++ '[' '!' -f /etc/ansible/ansible.cfg ']' ++ mkdir /etc/ansible ++ cat ++ ansible_version= ++ command -v ansible-playbook ++ '[' -z '' ']' ++ /usr/local/ghpc-venv/bin/python3 -m pip install ansible==8.7.0 +Collecting ansible==8.7.0 + Downloading ansible-8.7.0-py3-none-any.whl.metadata (7.9 kB) +Collecting ansible-core~=2.15.7 (from ansible==8.7.0) + Downloading ansible_core-2.15.13-py3-none-any.whl.metadata (7.0 kB) +Collecting jinja2>=3.0.0 (from ansible-core~=2.15.7->ansible==8.7.0) + Downloading jinja2-3.1.6-py3-none-any.whl.metadata (2.9 kB) +Collecting PyYAML>=5.1 (from ansible-core~=2.15.7->ansible==8.7.0) + Downloading PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (2.1 kB) +Collecting cryptography (from ansible-core~=2.15.7->ansible==8.7.0) + Downloading cryptography-45.0.5-cp311-abi3-manylinux_2_28_x86_64.whl.metadata (5.7 kB) +Collecting packaging (from ansible-core~=2.15.7->ansible==8.7.0) + Downloading packaging-25.0-py3-none-any.whl.metadata (3.3 kB) +Collecting resolvelib<1.1.0,>=0.5.3 (from ansible-core~=2.15.7->ansible==8.7.0) + Downloading resolvelib-1.0.1-py2.py3-none-any.whl.metadata (4.0 kB) +Collecting MarkupSafe>=2.0 (from jinja2>=3.0.0->ansible-core~=2.15.7->ansible==8.7.0) + Downloading MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.0 kB) +Collecting cffi>=1.14 (from cryptography->ansible-core~=2.15.7->ansible==8.7.0) + Downloading cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.5 kB) +Collecting pycparser (from cffi>=1.14->cryptography->ansible-core~=2.15.7->ansible==8.7.0) + Downloading pycparser-2.22-py3-none-any.whl.metadata (943 bytes) +Downloading ansible-8.7.0-py3-none-any.whl (48.4 MB) + ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā” 48.4/48.4 MB 102.4 MB/s eta 0:00:00 +Downloading ansible_core-2.15.13-py3-none-any.whl (2.3 MB) + ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā” 2.3/2.3 MB 105.5 MB/s eta 0:00:00 +Downloading resolvelib-1.0.1-py2.py3-none-any.whl (17 kB) +Downloading jinja2-3.1.6-py3-none-any.whl (134 kB) +Downloading MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (23 kB) +Downloading PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (767 kB) + ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā” 767.5/767.5 kB 59.6 MB/s eta 0:00:00 +Downloading cryptography-45.0.5-cp311-abi3-manylinux_2_28_x86_64.whl (4.5 MB) + ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā” 4.5/4.5 MB 153.6 MB/s eta 0:00:00 +Downloading cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (479 kB) +Downloading packaging-25.0-py3-none-any.whl (66 kB) +Downloading pycparser-2.22-py3-none-any.whl (117 kB) +Installing collected packages: resolvelib, PyYAML, pycparser, packaging, MarkupSafe, jinja2, cffi, cryptography, ansible-core, ansible + +Successfully installed MarkupSafe-3.0.2 PyYAML-6.0.2 ansible-8.7.0 ansible-core-2.15.13 cffi-1.17.1 cryptography-45.0.5 jinja2-3.1.6 packaging-25.0 pycparser-2.22 resolvelib-1.0.1 ++ read -r cmd ++ '[' -L /usr/bin/ansible ']' ++ ln -s /usr/local/ghpc-venv/bin/ansible /usr/bin/ansible ++ read -r cmd ++ '[' -L /usr/bin/ansible-config ']' ++ ln -s /usr/local/ghpc-venv/bin/ansible-config /usr/bin/ansible-config ++ read -r cmd ++ '[' -L /usr/bin/ansible-connection ']' ++ ln -s /usr/local/ghpc-venv/bin/ansible-connection /usr/bin/ansible-connection ++ read -r cmd ++ '[' -L /usr/bin/ansible-console ']' ++ ln -s /usr/local/ghpc-venv/bin/ansible-console /usr/bin/ansible-console ++ read -r cmd ++ '[' -L /usr/bin/ansible-doc ']' ++ ln -s /usr/local/ghpc-venv/bin/ansible-doc /usr/bin/ansible-doc ++ read -r cmd ++ '[' -L /usr/bin/ansible-galaxy ']' ++ ln -s /usr/local/ghpc-venv/bin/ansible-galaxy /usr/bin/ansible-galaxy ++ read -r cmd ++ '[' -L /usr/bin/ansible-inventory ']' ++ ln -s /usr/local/ghpc-venv/bin/ansible-inventory /usr/bin/ansible-inventory ++ read -r cmd ++ '[' -L /usr/bin/ansible-playbook ']' ++ ln -s /usr/local/ghpc-venv/bin/ansible-playbook /usr/bin/ansible-playbook ++ read -r cmd ++ '[' -L /usr/bin/ansible-pull ']' ++ ln -s /usr/local/ghpc-venv/bin/ansible-pull /usr/bin/ansible-pull ++ read -r cmd ++ '[' -L /usr/bin/ansible-test ']' ++ ln -s /usr/local/ghpc-venv/bin/ansible-test /usr/bin/ansible-test ++ read -r cmd ++ '[' -L /usr/bin/ansible-vault ']' ++ ln -s /usr/local/ghpc-venv/bin/ansible-vault /usr/bin/ansible-vault ++ read -r cmd +Thu Jul 24 15:25:55 +0000 2025 Info [1870]: === install_ansible_automatic.sh-ccf1 finished with exit_code=0 === +Thu Jul 24 15:25:57 +0000 2025 Info [1870]: === start executing runner: ansible_docker_settings.json-c21f === +Thu Jul 24 15:25:57 +0000 2025 Info [1870]: === ansible_docker_settings.json-c21f finished with exit_code=0 === +Thu Jul 24 15:25:58 +0000 2025 Info [1870]: === start executing runner: install_docker.yml-b4a8 === + +PLAY [Install and configure Docker] ******************************************** + +TASK [Gathering Facts] ********************************************************* +ok: [localhost] + +TASK [Gather distribution facts] *********************************************** +ok: [localhost] + +TASK [Check if docker is installed] ******************************************** +ok: [localhost] + +TASK [Add Docker GPG key (Rocky)] ********************************************** +changed: [localhost] + +TASK [Add Docker repository for Rocky Linux] *********************************** +changed: [localhost] + +TASK [Install Docker Engine and related packages on Rocky] ********************* +changed: [localhost] + +TASK [Install python3-pip] ***************************************************** +ok: [localhost] + +TASK [Install Docker SDK for Python] ******************************************* +changed: [localhost] + +TASK [Download Docker Installer (get.docker.com)] ****************************** +skipping: [localhost] + +TASK [Install Docker] ********************************************************** +skipping: [localhost] + +TASK [Create Docker daemon configuration] ************************************** +skipping: [localhost] + +TASK [Create Docker service override directory] ******************************** +changed: [localhost] + +TASK [Create Docker service override configuration] **************************** +changed: [localhost] + +TASK [Create Docker socket override directory] ********************************* +skipping: [localhost] + +TASK [Create Docker socket override configuration] ***************************** +skipping: [localhost] + +TASK [Delete Docker socket override configuration] ***************************** +ok: [localhost] + +RUNNING HANDLER [Refresh dnf cache] ******************************************** +ok: [localhost] + +TASK [Start Docker] ************************************************************ +changed: [localhost] + +PLAY RECAP ********************************************************************* +localhost : ok=13 changed=7 unreachable=0 failed=0 skipped=5 rescued=0 ignored=0 + +Thu Jul 24 15:27:29 +0000 2025 Info [1870]: === install_docker.yml-b4a8 finished with exit_code=0 === +Thu Jul 24 15:27:31 +0000 2025 Info [1870]: === start executing runner: install-deps.sh-1526 === ++ /usr/local/ghpc-venv/bin/python3 -m pip install requests google-auth docker +Requirement already satisfied: requests in /usr/local/ghpc-venv/lib64/python3.12/site-packages (2.32.4) +Collecting google-auth + Downloading google_auth-2.40.3-py2.py3-none-any.whl.metadata (6.2 kB) +Requirement already satisfied: docker in /usr/local/ghpc-venv/lib64/python3.12/site-packages (7.1.0) +Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/ghpc-venv/lib64/python3.12/site-packages (from requests) (3.4.2) +Requirement already satisfied: idna<4,>=2.5 in /usr/local/ghpc-venv/lib64/python3.12/site-packages (from requests) (3.10) +Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/ghpc-venv/lib64/python3.12/site-packages (from requests) (2.5.0) +Requirement already satisfied: certifi>=2017.4.17 in /usr/local/ghpc-venv/lib64/python3.12/site-packages (from requests) (2025.7.14) +Collecting cachetools<6.0,>=2.0.0 (from google-auth) + Downloading cachetools-5.5.2-py3-none-any.whl.metadata (5.4 kB) +Collecting pyasn1-modules>=0.2.1 (from google-auth) + Downloading pyasn1_modules-0.4.2-py3-none-any.whl.metadata (3.5 kB) +Collecting rsa<5,>=3.1.4 (from google-auth) + Downloading rsa-4.9.1-py3-none-any.whl.metadata (5.6 kB) +Collecting pyasn1>=0.1.3 (from rsa<5,>=3.1.4->google-auth) + Downloading pyasn1-0.6.1-py3-none-any.whl.metadata (8.4 kB) +Downloading google_auth-2.40.3-py2.py3-none-any.whl (216 kB) +Downloading cachetools-5.5.2-py3-none-any.whl (10 kB) +Downloading rsa-4.9.1-py3-none-any.whl (34 kB) +Downloading pyasn1-0.6.1-py3-none-any.whl (83 kB) +Downloading pyasn1_modules-0.4.2-py3-none-any.whl (181 kB) +Installing collected packages: pyasn1, cachetools, rsa, pyasn1-modules, google-auth + +Successfully installed cachetools-5.5.2 google-auth-2.40.3 pyasn1-0.6.1 pyasn1-modules-0.4.2 rsa-4.9.1 ++ ansible-galaxy collection install google.cloud +Starting galaxy collection install process +Nothing to do. All requested collections are already installed. If you want to reinstall them, consider using `--force`. +Thu Jul 24 15:27:32 +0000 2025 Info [1870]: === install-deps.sh-1526 finished with exit_code=0 === +Thu Jul 24 15:27:34 +0000 2025 Info [1870]: === start executing runner: roles.tar.gz-d489 === +Thu Jul 24 15:27:34 +0000 2025 Info [1870]: === roles.tar.gz-d489 finished with exit_code=0 === +Thu Jul 24 15:27:36 +0000 2025 Info [1870]: === start executing runner: unpack_roles.sh-994e === ++ mkdir -p /tmp/vdi/roles ++ tar xzf /tmp/vdi/roles.tar.gz -C /tmp/vdi/roles +Thu Jul 24 15:27:36 +0000 2025 Info [1870]: === unpack_roles.sh-994e finished with exit_code=0 === +Thu Jul 24 15:27:38 +0000 2025 Info [1870]: === start executing runner: vars.yaml-faf2 === +Thu Jul 24 15:27:38 +0000 2025 Info [1870]: === vars.yaml-faf2 finished with exit_code=0 === +Thu Jul 24 15:27:40 +0000 2025 Info [1870]: === start executing runner: install.yaml-f44d === +Using /etc/ansible/ansible.cfg as config file + +PLAY [localhost] *************************************************************** + +TASK [Gathering Facts] ********************************************************* +ok: [localhost] + +TASK [lock_manager : Check if lock file exists] ******************************** +ok: [localhost] => { + "changed": false, + "stat": { + "exists": false + } +} + +TASK [lock_manager : Debug lock file existence] ******************************** +ok: [localhost] => {} + +MSG: + +Lock file exists: False + +TASK [lock_manager : Load existing lock file] ********************************** +skipping: [localhost] => { + "changed": false, + "false_condition": "lock_file_stat.stat.exists", + "skip_reason": "Conditional result was False" +} + +TASK [lock_manager : Debug lock file load result] ****************************** +skipping: [localhost] => { + "false_condition": "debug | default(false) and lock_file_stat.stat.exists" +} + +TASK [lock_manager : Calculate current deployment hash] ************************ +ok: [localhost] => { + "ansible_facts": { + "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab" + }, + "changed": false +} + +TASK [lock_manager : Debug deployment hash calculation] ************************ +ok: [localhost] => {} + +MSG: + +Current deployment hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab +Deployment components: + - deployment_name: vdi-test-scott + - project_id: hpc-discovery-external + - vdi_tool: guacamole + - vnc_flavor: tigervnc + - user_provision: local_users + - vdi_webapp_port: 8081 + - vdi_resolution: 1920x1080 + - vdi_users count: 2 + + +TASK [lock_manager : Calculate current user secrets hash] ********************** +ok: [localhost] => { + "ansible_facts": { + "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" + }, + "changed": false +} + +TASK [lock_manager : Debug user secrets hash calculation] ********************** +ok: [localhost] => {} + +MSG: + +Current user secrets hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 +User configuration components: + - vdi_users (blueprint): [{"username": "alice", "port": 5901}, {"username": "bob", "port": 5902, "secret_name": "a-password-for-bob"}] + + +TASK [lock_manager : Set default values for missing lock file] ***************** +ok: [localhost] => { + "ansible_facts": { + "vdi_setup_status": { + "completed_roles": { + "base_os": { + "completed": false + }, + "lock_manager": { + "completed": false + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "none", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:27:41Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [], + "previous_users": [], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:27:41Z", + "user_secrets_hash": "none", + "users_updated": [] + } + } + }, + "changed": false +} + +TASK [lock_manager : Debug lock file status] *********************************** +ok: [localhost] => {} + +MSG: + +Lock file status: + - exists: False + - deployment_hash: none + - user_secrets_hash: none + - force_rerun: False + + +TASK [lock_manager : Check if lock file exists] ******************************** +ok: [localhost] => { + "ansible_facts": { + "lock_file_exists": false + }, + "changed": false +} + +TASK [lock_manager : Check if current role is completed] *********************** +ok: [localhost] => { + "ansible_facts": { + "current_role_completed": false + }, + "changed": false +} + +TASK [lock_manager : Check if deployment hash matches] ************************* +ok: [localhost] => { + "ansible_facts": { + "deployment_hash_matches": false + }, + "changed": false +} + +TASK [lock_manager : Check if force rerun is enabled] ************************** +ok: [localhost] => { + "ansible_facts": { + "force_rerun_enabled": false + }, + "changed": false +} + +TASK [lock_manager : Determine if role should run] ***************************** +ok: [localhost] => { + "ansible_facts": { + "role_should_run": true + }, + "changed": false +} + +TASK [lock_manager : Debug role should run calculation] ************************ +ok: [localhost] => {} + +MSG: + +Role should run calculation for lock_manager: + - not lock_file_exists: True + - not current_role_completed: True + - not deployment_hash_matches: True + - force_rerun_enabled: False + - role_should_run: True + + +TASK [lock_manager : Debug role execution decision] **************************** +ok: [localhost] => {} + +MSG: + +Role execution decision for lock_manager: + - current_role: not set + - ansible_role_name: lock_manager + - role_should_run: True + - lock_file_exists: False + - current_role_completed: False + - deployment_hash_matches: False + - force_rerun_enabled: False + + +TASK [lock_manager : Check if user configuration has changed] ****************** +ok: [localhost] => { + "ansible_facts": { + "user_config_changed": true + }, + "changed": false +} + +TASK [lock_manager : Set user secrets changed flag (will be updated by secret_manager role)] *** +ok: [localhost] => { + "ansible_facts": { + "user_secrets_changed": true + }, + "changed": false +} + +TASK [lock_manager : Debug user secrets change detection] ********************** +ok: [localhost] => {} + +MSG: + +User configuration change detection: + - user_config_changed: True + - user_secrets_changed: True + - stored_hash: none + - current_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 + + +TASK [lock_manager : Set lock check result] ************************************ +ok: [localhost] => { + "ansible_facts": { + "lock_check_result": { + "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "existing_deployment_hash": "none", + "existing_user_secrets_hash": "none", + "force_rerun": false, + "role_should_run": true, + "user_secrets_changed": true + } + }, + "changed": false +} + +TASK [lock_manager : Debug final lock check result] **************************** +ok: [localhost] => {} + +MSG: + +Final lock check result: + - role_should_run: True + - deployment_hash_changed: True + - user_secrets_changed: True + - force_rerun: False + + +TASK [lock_manager : Set current role (only if not already set)] *************** +ok: [localhost] => { + "ansible_facts": { + "current_role": "lock_manager" + }, + "changed": false +} + +TASK [lock_manager : Debug lock manager role execution] ************************ +ok: [localhost] => {} + +MSG: + +Lock manager role started - operation: check for role: lock_manager + +TASK [lock_manager : Ensure lock directory exists] ***************************** +changed: [localhost] => { + "changed": true, + "gid": 0, + "group": "root", + "mode": "0755", + "owner": "root", + "path": "/opt/vdi-setup", + "size": 6, + "state": "directory", + "uid": 0 +} + +TASK [lock_manager : Debug lock directory status] ****************************** +ok: [localhost] => {} + +MSG: + +Lock directory created: True + +TASK [lock_manager : Include check lock tasks] ********************************* +included: /tmp/vdi/roles/lock_manager/tasks/check_lock.yaml for localhost + +TASK [lock_manager : Check if lock file exists] ******************************** +ok: [localhost] => { + "changed": false, + "stat": { + "exists": false + } +} + +TASK [lock_manager : Debug lock file existence] ******************************** +ok: [localhost] => {} + +MSG: + +Lock file exists: False + +TASK [lock_manager : Load existing lock file] ********************************** +skipping: [localhost] => { + "changed": false, + "false_condition": "lock_file_stat.stat.exists", + "skip_reason": "Conditional result was False" +} + +TASK [lock_manager : Debug lock file load result] ****************************** +skipping: [localhost] => { + "false_condition": "debug | default(false) and lock_file_stat.stat.exists" +} + +TASK [lock_manager : Calculate current deployment hash] ************************ +ok: [localhost] => { + "ansible_facts": { + "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab" + }, + "changed": false +} + +TASK [lock_manager : Debug deployment hash calculation] ************************ +ok: [localhost] => {} + +MSG: + +Current deployment hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab +Deployment components: + - deployment_name: vdi-test-scott + - project_id: hpc-discovery-external + - vdi_tool: guacamole + - vnc_flavor: tigervnc + - user_provision: local_users + - vdi_webapp_port: 8081 + - vdi_resolution: 1920x1080 + - vdi_users count: 2 + + +TASK [lock_manager : Calculate current user secrets hash] ********************** +ok: [localhost] => { + "ansible_facts": { + "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" + }, + "changed": false +} + +TASK [lock_manager : Debug user secrets hash calculation] ********************** +ok: [localhost] => {} + +MSG: + +Current user secrets hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 +User configuration components: + - vdi_users (blueprint): [{"username": "alice", "port": 5901}, {"username": "bob", "port": 5902, "secret_name": "a-password-for-bob"}] + + +TASK [lock_manager : Set default values for missing lock file] ***************** +ok: [localhost] => { + "ansible_facts": { + "vdi_setup_status": { + "completed_roles": { + "base_os": { + "completed": false + }, + "lock_manager": { + "completed": false + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "none", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:27:41Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [], + "previous_users": [], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:27:41Z", + "user_secrets_hash": "none", + "users_updated": [] + } + } + }, + "changed": false +} + +TASK [lock_manager : Debug lock file status] *********************************** +ok: [localhost] => {} + +MSG: + +Lock file status: + - exists: False + - deployment_hash: none + - user_secrets_hash: none + - force_rerun: False + + +TASK [lock_manager : Check if lock file exists] ******************************** +ok: [localhost] => { + "ansible_facts": { + "lock_file_exists": false + }, + "changed": false +} + +TASK [lock_manager : Check if current role is completed] *********************** +ok: [localhost] => { + "ansible_facts": { + "current_role_completed": false + }, + "changed": false +} + +TASK [lock_manager : Check if deployment hash matches] ************************* +ok: [localhost] => { + "ansible_facts": { + "deployment_hash_matches": false + }, + "changed": false +} + +TASK [lock_manager : Check if force rerun is enabled] ************************** +ok: [localhost] => { + "ansible_facts": { + "force_rerun_enabled": false + }, + "changed": false +} + +TASK [lock_manager : Determine if role should run] ***************************** +ok: [localhost] => { + "ansible_facts": { + "role_should_run": true + }, + "changed": false +} + +TASK [lock_manager : Debug role should run calculation] ************************ +ok: [localhost] => {} + +MSG: + +Role should run calculation for lock_manager: + - not lock_file_exists: True + - not current_role_completed: True + - not deployment_hash_matches: True + - force_rerun_enabled: False + - role_should_run: True + + +TASK [lock_manager : Debug role execution decision] **************************** +ok: [localhost] => {} + +MSG: + +Role execution decision for lock_manager: + - current_role: lock_manager + - ansible_role_name: lock_manager + - role_should_run: True + - lock_file_exists: False + - current_role_completed: False + - deployment_hash_matches: False + - force_rerun_enabled: False + + +TASK [lock_manager : Check if user configuration has changed] ****************** +ok: [localhost] => { + "ansible_facts": { + "user_config_changed": true + }, + "changed": false +} + +TASK [lock_manager : Set user secrets changed flag (will be updated by secret_manager role)] *** +ok: [localhost] => { + "ansible_facts": { + "user_secrets_changed": true + }, + "changed": false +} + +TASK [lock_manager : Debug user secrets change detection] ********************** +ok: [localhost] => {} + +MSG: + +User configuration change detection: + - user_config_changed: True + - user_secrets_changed: True + - stored_hash: none + - current_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 + + +TASK [lock_manager : Set lock check result] ************************************ +ok: [localhost] => { + "ansible_facts": { + "lock_check_result": { + "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "existing_deployment_hash": "none", + "existing_user_secrets_hash": "none", + "force_rerun": false, + "role_should_run": true, + "user_secrets_changed": true + } + }, + "changed": false +} + +TASK [lock_manager : Debug final lock check result] **************************** +ok: [localhost] => {} + +MSG: + +Final lock check result: + - role_should_run: True + - deployment_hash_changed: True + - user_secrets_changed: True + - force_rerun: False + + +TASK [lock_manager : Include create lock tasks] ******************************** +skipping: [localhost] => { + "changed": false, + "false_condition": "lock_operation | default('check') == 'create'", + "skip_reason": "Conditional result was False" +} + +TASK [lock_manager : Include cleanup lock tasks] ******************************* +skipping: [localhost] => { + "changed": false, + "false_condition": "lock_operation | default('check') == 'cleanup'", + "skip_reason": "Conditional result was False" +} + +TASK [lock_manager : Debug lock manager role completion] *********************** +ok: [localhost] => {} + +MSG: + +Lock manager role completed - operation: check + +TASK [lock_manager : Mark lock_manager role as completed] ********************** +included: /tmp/vdi/roles/lock_manager/tasks/create_lock.yaml for localhost + +TASK [lock_manager : Debug create lock operation] ****************************** +ok: [localhost] => {} + +MSG: + +Creating/updating lock file for role: lock_manager + +TASK [lock_manager : Check if lock file exists] ******************************** +ok: [localhost] => { + "changed": false, + "stat": { + "exists": false + } +} + +TASK [lock_manager : Debug lock file existence] ******************************** +ok: [localhost] => {} + +MSG: + +Lock file exists: False + +TASK [lock_manager : Create initial lock file structure] *********************** +changed: [localhost] => { + "changed": true, + "checksum": "8479c862b92b39583c151fca822f92954c92eee0", + "dest": "/opt/vdi-setup/.vdi-lock.yaml", + "gid": 0, + "group": "root", + "md5sum": "e3c95caac8aea2fba82a6d39dec216d8", + "mode": "0644", + "owner": "root", + "size": 909, + "src": "/root/.ansible/tmp/ansible-tmp-1753370864.1744716-6171-78949705975838/source", + "state": "file", + "uid": 0 +} + +TASK [lock_manager : Debug lock file creation] ********************************* +ok: [localhost] => {} + +MSG: + +Lock file created: True + +TASK [lock_manager : Load existing lock file data] ***************************** +ok: [localhost] => { + "ansible_facts": { + "vdi_setup_status": { + "completed_roles": { + "base_os": { + "completed": false + }, + "lock_manager": { + "completed": false + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:27:41Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [], + "previous_users": [], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:27:41Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [] + } + } + }, + "ansible_included_var_files": [ + "/opt/vdi-setup/.vdi-lock.yaml" + ], + "changed": false +} + +TASK [lock_manager : Get current lock data] ************************************ +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": false + }, + "lock_manager": { + "completed": false + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:27:41Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [], + "previous_users": [], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:27:41Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [] + } + } + }, + "changed": false +} + +TASK [lock_manager : Update deployment hash] *********************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": false + }, + "lock_manager": { + "completed": false + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:27:41Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [], + "previous_users": [], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:27:41Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [] + } + } + }, + "changed": false +} + +TASK [lock_manager : Get current timestamp] ************************************ +changed: [localhost] => { + "changed": true, + "cmd": [ + "date", + "-u", + "+%Y-%m-%dT%H:%M:%SZ" + ], + "delta": "0:00:00.002395", + "end": "2025-07-24 15:27:44.996404", + "rc": 0, + "start": "2025-07-24 15:27:44.994009" +} + +STDOUT: + +2025-07-24T15:27:44Z + +TASK [lock_manager : Update last updated timestamp] **************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": false + }, + "lock_manager": { + "completed": false + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:27:44Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [], + "previous_users": [], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:27:41Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [] + } + } + }, + "changed": false +} + +TASK [lock_manager : Get current user secrets status] ************************** +ok: [localhost] => { + "ansible_facts": { + "current_user_secrets": { + "last_secret_check": "2025-07-24T15:27:41Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [] + } + }, + "changed": false +} + +TASK [lock_manager : Create user secrets update] ******************************* +ok: [localhost] => { + "ansible_facts": { + "user_secrets_update": { + "last_secret_check": "2025-07-24T15:27:44Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" + } + }, + "changed": false +} + +TASK [lock_manager : Update user secrets hash] ********************************* +ok: [localhost] => { + "ansible_facts": { + "updated_user_secrets": { + "last_secret_check": "2025-07-24T15:27:44Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [] + } + }, + "changed": false +} + +TASK [lock_manager : Apply user secrets update] ******************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": false + }, + "lock_manager": { + "completed": false + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:27:44Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [], + "previous_users": [], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:27:44Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [] + } + } + }, + "changed": false +} + +TASK [lock_manager : Create role status entry] ********************************* +ok: [localhost] => { + "ansible_facts": { + "role_status_entry": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + } + }, + "changed": false +} + +TASK [lock_manager : Debug role status entry] ********************************** +ok: [localhost] => {} + +MSG: + +Role status entry for lock_manager: + - completed: True + - completed_at: 2025-07-24T15:27:44Z + + +TASK [lock_manager : Debug current lock data before role update] *************** +ok: [localhost] => {} + +MSG: + +Current lock data before updating lock_manager: + - current_lock_data: completed_roles: + base_os: + completed: false + lock_manager: + completed: false + secret_manager: + completed: false + user_provision: + completed: false + vdi_tool: + completed: false + vnc: + completed: false +created_at: '2025-07-24T15:27:41Z' +deployment_hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab +deployment_name: vdi-test-scott +force_rerun: false +last_updated: '2025-07-24T15:27:44Z' +lock_version: '1.0' +setup_status: configuring +user_management: + current_users: [] + previous_users: [] + removed_users: [] +user_secrets_status: + last_secret_check: '2025-07-24T15:27:44Z' + user_secrets_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 + users_updated: [] + + - current_completed_roles: base_os: + completed: false +lock_manager: + completed: false +secret_manager: + completed: false +user_provision: + completed: false +vdi_tool: + completed: false +vnc: + completed: false + + - role_completed: True + - current_role: lock_manager + + +TASK [lock_manager : Get current completed roles] ****************************** +ok: [localhost] => { + "ansible_facts": { + "current_completed_roles": { + "base_os": { + "completed": false + }, + "lock_manager": { + "completed": false + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + } + }, + "changed": false +} + +TASK [lock_manager : Update completed roles] *********************************** +ok: [localhost] => { + "ansible_facts": { + "updated_completed_roles": { + "base_os": { + "completed": false + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + } + }, + "changed": false +} + +TASK [lock_manager : Update lock data with completed roles] ******************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": false + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:27:44Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [], + "previous_users": [], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:27:44Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [] + } + } + }, + "changed": false +} + +TASK [lock_manager : Debug updated lock data after role update] **************** +ok: [localhost] => {} + +MSG: + +Updated lock data after updating lock_manager: + - current_role: lock_manager + - role_completed: True + - completed_roles: {'base_os': {'completed': False}, 'lock_manager': {'completed': True, 'completed_at': '2025-07-24T15:27:44Z'}, 'secret_manager': {'completed': False}, 'user_provision': {'completed': False}, 'vnc': {'completed': False}, 'vdi_tool': {'completed': False}} + + +TASK [lock_manager : Get current users updated list] *************************** +ok: [localhost] => { + "ansible_facts": { + "current_users_updated": [] + }, + "changed": false +} + +TASK [lock_manager : Get new usernames list] *********************************** +ok: [localhost] => { + "ansible_facts": { + "new_usernames": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Combine users updated lists] ****************************** +ok: [localhost] => { + "ansible_facts": { + "combined_users_updated": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Debug user deduplication] ********************************* +ok: [localhost] => {} + +MSG: + +User deduplication for lock_manager: + - current_users_updated: [] + - new_usernames: ['alice', 'bob'] + - combined_users_updated: ['alice', 'bob'] + + +TASK [lock_manager : Get current user secrets for list update] ***************** +ok: [localhost] => { + "ansible_facts": { + "current_user_secrets_for_list": { + "last_secret_check": "2025-07-24T15:27:44Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [] + } + }, + "changed": false +} + +TASK [lock_manager : Update users updated list] ******************************** +ok: [localhost] => { + "ansible_facts": { + "updated_user_secrets_with_list": { + "last_secret_check": "2025-07-24T15:27:44Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Apply users updated list] ********************************* +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": false + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:27:44Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [], + "previous_users": [], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:27:44Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Get current user management data] ************************* +ok: [localhost] => { + "ansible_facts": { + "current_user_management": { + "current_users": [], + "previous_users": [], + "removed_users": [] + } + }, + "changed": false +} + +TASK [lock_manager : Get current usernames list] ******************************* +ok: [localhost] => { + "ansible_facts": { + "current_usernames": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Update user management data] ****************************** +ok: [localhost] => { + "ansible_facts": { + "updated_user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [], + "removed_users": [] + } + }, + "changed": false +} + +TASK [lock_manager : Apply user management update] ***************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": false + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:27:44Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:27:44Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Debug user management] ************************************ +ok: [localhost] => {} + +MSG: + +User management for lock_manager: + - current_users: ['alice', 'bob'] + - previous_users: [] + - removed_users: [] + + +TASK [lock_manager : Check if all roles are completed] ************************* +ok: [localhost] => { + "ansible_facts": { + "all_roles_completed": false + }, + "changed": false +} + +TASK [lock_manager : Update setup status based on completion] ****************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": false + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:27:44Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:27:44Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Debug setup status update] ******************************** +ok: [localhost] => {} + +MSG: + +Setup status update: + - all_roles_completed: False + - setup_status: configuring + - completed_roles_count: 1 + - total_roles_count: 6 + + +TASK [lock_manager : Write updated lock file] ********************************** +changed: [localhost] => { + "changed": true, + "checksum": "358a7f4095fbb49a0eedafd77e97040d1b9b5a26", + "dest": "/opt/vdi-setup/.vdi-lock.yaml", + "gid": 0, + "group": "root", + "md5sum": "2cf874286e9bf3b17fc771283955c695", + "mode": "0644", + "owner": "root", + "size": 879, + "src": "/root/.ansible/tmp/ansible-tmp-1753370865.96913-6272-170962703062563/source", + "state": "file", + "uid": 0 +} + +TASK [lock_manager : Debug VM metadata update] ********************************* +ok: [localhost] => {} + +MSG: + +VM metadata update info: + - hostname: vdi-test-scott-rocky-0 + - zone: us-central1-a + - lock_file_changed: True + + +TASK [lock_manager : Get lock file content as base64] ************************** +changed: [localhost] => { + "changed": true, + "cmd": "cat /opt/vdi-setup/.vdi-lock.yaml | base64 -w 0", + "delta": "0:00:00.005528", + "end": "2025-07-24 15:27:46.510774", + "rc": 0, + "start": "2025-07-24 15:27:46.505246" +} + +STDOUT: + +dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IGZhbHNlCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IGZhbHNlCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiBmYWxzZQogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogZmFsc2UKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiBmYWxzZQogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNToyNzo0NFonCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogY29uZmlndXJpbmcKICB1c2VyX21hbmFnZW1lbnQ6CiAgICBjdXJyZW50X3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHByZXZpb3VzX3VzZXJzOiBbXQogICAgcmVtb3ZlZF91c2VyczogW10KICB1c2VyX3NlY3JldHNfc3RhdHVzOgogICAgbGFzdF9zZWNyZXRfY2hlY2s6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHVzZXJfc2VjcmV0c19oYXNoOiA2YTZhMDJkM2QwOTkzM2NhMjgyYWUyYjAyMzM2MzQzYzlkNzM3MDFlNDViZDgxNDc1ZmJkOTJkY2Y5ZDJmM2I2CiAgICB1c2Vyc191cGRhdGVkOgogICAgLSBhbGljZQogICAgLSBib2IK + +TASK [lock_manager : Add VDI lock file content to VM metadata] ***************** +changed: [localhost] => { + "changed": true, + "cmd": [ + "gcloud", + "compute", + "instances", + "add-metadata", + "vdi-test-scott-rocky-0", + "--metadata", + "vdi-lock-content=dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IGZhbHNlCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IGZhbHNlCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiBmYWxzZQogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogZmFsc2UKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiBmYWxzZQogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNToyNzo0NFonCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogY29uZmlndXJpbmcKICB1c2VyX21hbmFnZW1lbnQ6CiAgICBjdXJyZW50X3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHByZXZpb3VzX3VzZXJzOiBbXQogICAgcmVtb3ZlZF91c2VyczogW10KICB1c2VyX3NlY3JldHNfc3RhdHVzOgogICAgbGFzdF9zZWNyZXRfY2hlY2s6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHVzZXJfc2VjcmV0c19oYXNoOiA2YTZhMDJkM2QwOTkzM2NhMjgyYWUyYjAyMzM2MzQzYzlkNzM3MDFlNDViZDgxNDc1ZmJkOTJkY2Y5ZDJmM2I2CiAgICB1c2Vyc191cGRhdGVkOgogICAgLSBhbGljZQogICAgLSBib2IK", + "--zone=us-central1-a" + ], + "delta": "0:00:04.460963", + "end": "2025-07-24 15:27:51.159269", + "failed_when_result": false, + "rc": 0, + "start": "2025-07-24 15:27:46.698306" +} + +STDERR: + +Updated [https://www.googleapis.com/compute/v1/projects/hpc-discovery-external/zones/us-central1-a/instances/vdi-test-scott-rocky-0]. + +TASK [lock_manager : Debug lock file write] ************************************ +ok: [localhost] => {} + +MSG: + +Lock file written: True + +TASK [lock_manager : Verify lock file integrity] ******************************* +ok: [localhost] => { + "changed": false, + "cmd": [ + "python3", + "-c", + "import yaml; yaml.safe_load(open('/opt/vdi-setup/.vdi-lock.yaml'))" + ], + "delta": "0:00:00.050900", + "end": "2025-07-24 15:27:51.433067", + "rc": 0, + "start": "2025-07-24 15:27:51.382167" +} + +TASK [lock_manager : Debug lock file validation] ******************************* +ok: [localhost] => {} + +MSG: + +Lock file validation: True + +TASK [lock_manager : Check if VDI monitor service exists] ********************** +ok: [localhost] => { + "changed": false, + "stat": { + "exists": false + } +} + +TASK [lock_manager : Debug VDI monitor setup conditions] *********************** +ok: [localhost] => {} + +MSG: + +VDI monitor setup conditions: + - current_role: lock_manager + - vdi_monitor_service_exists: False + - lock_file_creation.changed: True + - will_setup_monitor: True + + +TASK [lock_manager : Setup VDI monitoring service (first time only)] *********** +included: /tmp/vdi/roles/lock_manager/tasks/monitor_setup.yaml for localhost + +TASK [lock_manager : Create VDI monitor script directory] ********************** +ok: [localhost] => { + "changed": false, + "gid": 0, + "group": "root", + "mode": "0755", + "owner": "root", + "path": "/usr/local/bin", + "size": 6, + "state": "directory", + "uid": 0 +} + +TASK [lock_manager : Copy VDI monitor script] ********************************** +changed: [localhost] => { + "changed": true, + "checksum": "49f5302fb57094d20c194a1f7379da9d16209de7", + "dest": "/usr/local/bin/vdi-monitor.sh", + "gid": 0, + "group": "root", + "md5sum": "62d4f0ae8c6876764d097330ba1393cb", + "mode": "0755", + "owner": "root", + "size": 5348, + "src": "/root/.ansible/tmp/ansible-tmp-1753370871.9519365-6406-123074483660618/source", + "state": "file", + "uid": 0 +} + +TASK [lock_manager : Copy VDI monitor systemd service] ************************* +changed: [localhost] => { + "changed": true, + "checksum": "f6de22707f8d9248014f679a2f1930451e2bf635", + "dest": "/etc/systemd/system/vdi-monitor.service", + "gid": 0, + "group": "root", + "md5sum": "03b2639860cf4c6a87c36ae7f885909d", + "mode": "0644", + "owner": "root", + "size": 283, + "src": "/root/.ansible/tmp/ansible-tmp-1753370872.3131151-6428-31276884095225/source", + "state": "file", + "uid": 0 +} + +TASK [lock_manager : Create VDI monitor log directory] ************************* +ok: [localhost] => { + "changed": false, + "gid": 0, + "group": "root", + "mode": "0755", + "owner": "root", + "path": "/var/log", + "size": 4096, + "state": "directory", + "uid": 0 +} + +TASK [lock_manager : Create VDI monitor log files] ***************************** +changed: [localhost] => (item=/var/log/vdi-monitor.log) => { + "ansible_loop_var": "item", + "changed": true, + "dest": "/var/log/vdi-monitor.log", + "gid": 0, + "group": "root", + "item": "/var/log/vdi-monitor.log", + "mode": "0644", + "owner": "root", + "size": 0, + "state": "file", + "uid": 0 +} +changed: [localhost] => (item=/var/log/ansible-vdi-reconfig.log) => { + "ansible_loop_var": "item", + "changed": true, + "dest": "/var/log/ansible-vdi-reconfig.log", + "gid": 0, + "group": "root", + "item": "/var/log/ansible-vdi-reconfig.log", + "mode": "0644", + "owner": "root", + "size": 0, + "state": "file", + "uid": 0 +} + +TASK [lock_manager : Reload systemd daemon] ************************************ +ok: [localhost] => { + "changed": false, + "name": null, + "status": {} +} + +TASK [lock_manager : Enable VDI monitor service] ******************************* +changed: [localhost] => { + "changed": true, + "enabled": true, + "name": "vdi-monitor", + "state": "started", + "status": { + "ActiveEnterTimestampMonotonic": "0", + "ActiveExitTimestampMonotonic": "0", + "ActiveState": "inactive", + "After": "systemd-journald.socket network.target system.slice sysinit.target basic.target", + "AllowIsolate": "no", + "AllowedCPUs": "", + "AllowedMemoryNodes": "", + "AmbientCapabilities": "", + "AssertResult": "no", + "AssertTimestampMonotonic": "0", + "Before": "shutdown.target", + "BlockIOAccounting": "no", + "BlockIOWeight": "[not set]", + "CPUAccounting": "no", + "CPUAffinity": "", + "CPUAffinityFromNUMA": "no", + "CPUQuotaPerSecUSec": "infinity", + "CPUQuotaPeriodUSec": "infinity", + "CPUSchedulingPolicy": "0", + "CPUSchedulingPriority": "0", + "CPUSchedulingResetOnFork": "no", + "CPUShares": "[not set]", + "CPUUsageNSec": "[not set]", + "CPUWeight": "[not set]", + "CacheDirectoryMode": "0755", + "CanFreeze": "yes", + "CanIsolate": "no", + "CanReload": "no", + "CanStart": "yes", + "CanStop": "yes", + "CapabilityBoundingSet": "cap_chown cap_dac_override cap_dac_read_search cap_fowner cap_fsetid cap_kill cap_setgid cap_setuid cap_setpcap cap_linux_immutable cap_net_bind_service cap_net_broadcast cap_net_admin cap_net_raw cap_ipc_lock cap_ipc_owner cap_sys_module cap_sys_rawio cap_sys_chroot cap_sys_ptrace cap_sys_pacct cap_sys_admin cap_sys_boot cap_sys_nice cap_sys_resource cap_sys_time cap_sys_tty_config cap_mknod cap_lease cap_audit_write cap_audit_control cap_setfcap cap_mac_override cap_mac_admin cap_syslog cap_wake_alarm cap_block_suspend cap_audit_read cap_perfmon cap_bpf", + "CollectMode": "inactive", + "ConditionResult": "no", + "ConditionTimestampMonotonic": "0", + "ConfigurationDirectoryMode": "0755", + "Conflicts": "shutdown.target", + "ControlPID": "0", + "DefaultDependencies": "yes", + "DefaultMemoryLow": "0", + "DefaultMemoryMin": "0", + "Delegate": "no", + "Description": "VDI Configuration Monitor", + "DevicePolicy": "auto", + "DynamicUser": "no", + "EffectiveCPUs": "", + "EffectiveMemoryNodes": "", + "ExecMainCode": "0", + "ExecMainExitTimestampMonotonic": "0", + "ExecMainPID": "0", + "ExecMainStartTimestampMonotonic": "0", + "ExecMainStatus": "0", + "ExecStart": "{ path=/usr/local/bin/vdi-monitor.sh ; argv[]=/usr/local/bin/vdi-monitor.sh ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", + "FailureAction": "none", + "FileDescriptorStoreMax": "0", + "FragmentPath": "/etc/systemd/system/vdi-monitor.service", + "FreezerState": "running", + "GID": "[not set]", + "Group": "root", + "GuessMainPID": "yes", + "IOAccounting": "no", + "IOSchedulingClass": "0", + "IOSchedulingPriority": "0", + "IOWeight": "[not set]", + "IPAccounting": "no", + "IPEgressBytes": "18446744073709551615", + "IPEgressPackets": "18446744073709551615", + "IPIngressBytes": "18446744073709551615", + "IPIngressPackets": "18446744073709551615", + "Id": "vdi-monitor.service", + "IgnoreOnIsolate": "no", + "IgnoreSIGPIPE": "yes", + "InactiveEnterTimestampMonotonic": "0", + "InactiveExitTimestampMonotonic": "0", + "JobRunningTimeoutUSec": "infinity", + "JobTimeoutAction": "none", + "JobTimeoutUSec": "infinity", + "KeyringMode": "private", + "KillMode": "control-group", + "KillSignal": "15", + "LimitAS": "infinity", + "LimitASSoft": "infinity", + "LimitCORE": "infinity", + "LimitCORESoft": "0", + "LimitCPU": "infinity", + "LimitCPUSoft": "infinity", + "LimitDATA": "infinity", + "LimitDATASoft": "infinity", + "LimitFSIZE": "infinity", + "LimitFSIZESoft": "infinity", + "LimitLOCKS": "infinity", + "LimitLOCKSSoft": "infinity", + "LimitMEMLOCK": "65536", + "LimitMEMLOCKSoft": "65536", + "LimitMSGQUEUE": "819200", + "LimitMSGQUEUESoft": "819200", + "LimitNICE": "0", + "LimitNICESoft": "0", + "LimitNOFILE": "262144", + "LimitNOFILESoft": "1024", + "LimitNPROC": "127383", + "LimitNPROCSoft": "127383", + "LimitRSS": "infinity", + "LimitRSSSoft": "infinity", + "LimitRTPRIO": "0", + "LimitRTPRIOSoft": "0", + "LimitRTTIME": "infinity", + "LimitRTTIMESoft": "infinity", + "LimitSIGPENDING": "127383", + "LimitSIGPENDINGSoft": "127383", + "LimitSTACK": "infinity", + "LimitSTACKSoft": "8388608", + "LoadState": "loaded", + "LockPersonality": "no", + "LogLevelMax": "-1", + "LogRateLimitBurst": "0", + "LogRateLimitIntervalUSec": "0", + "LogsDirectoryMode": "0755", + "MainPID": "0", + "MemoryAccounting": "yes", + "MemoryCurrent": "[not set]", + "MemoryDenyWriteExecute": "no", + "MemoryHigh": "infinity", + "MemoryLimit": "infinity", + "MemoryLow": "0", + "MemoryMax": "infinity", + "MemoryMin": "0", + "MemorySwapMax": "infinity", + "MountAPIVFS": "no", + "MountFlags": "", + "NFileDescriptorStore": "0", + "NRestarts": "0", + "NUMAMask": "", + "NUMAPolicy": "n/a", + "Names": "vdi-monitor.service", + "NeedDaemonReload": "no", + "Nice": "0", + "NoNewPrivileges": "no", + "NonBlocking": "no", + "NotifyAccess": "none", + "OOMScoreAdjust": "0", + "OnFailureJobMode": "replace", + "PermissionsStartOnly": "no", + "Perpetual": "no", + "PrivateDevices": "no", + "PrivateMounts": "no", + "PrivateNetwork": "no", + "PrivateTmp": "no", + "PrivateUsers": "no", + "ProtectControlGroups": "no", + "ProtectHome": "no", + "ProtectKernelModules": "no", + "ProtectKernelTunables": "no", + "ProtectSystem": "no", + "RefuseManualStart": "no", + "RefuseManualStop": "no", + "RemainAfterExit": "no", + "RemoveIPC": "no", + "Requires": "sysinit.target system.slice", + "Restart": "always", + "RestartUSec": "30s", + "RestrictNamespaces": "no", + "RestrictRealtime": "no", + "RestrictSUIDSGID": "no", + "Result": "success", + "RootDirectoryStartOnly": "no", + "RuntimeDirectoryMode": "0755", + "RuntimeDirectoryPreserve": "no", + "RuntimeMaxUSec": "infinity", + "SameProcessGroup": "no", + "SecureBits": "0", + "SendSIGHUP": "no", + "SendSIGKILL": "yes", + "Slice": "system.slice", + "StandardError": "journal", + "StandardInput": "null", + "StandardInputData": "", + "StandardOutput": "journal", + "StartLimitAction": "none", + "StartLimitBurst": "5", + "StartLimitIntervalUSec": "10s", + "StartupBlockIOWeight": "[not set]", + "StartupCPUShares": "[not set]", + "StartupCPUWeight": "[not set]", + "StartupIOWeight": "[not set]", + "StateChangeTimestampMonotonic": "0", + "StateDirectoryMode": "0755", + "StatusErrno": "0", + "StopWhenUnneeded": "no", + "SubState": "dead", + "SuccessAction": "none", + "SyslogFacility": "3", + "SyslogLevel": "6", + "SyslogLevelPrefix": "yes", + "SyslogPriority": "30", + "SystemCallErrorNumber": "0", + "TTYReset": "no", + "TTYVHangup": "no", + "TTYVTDisallocate": "no", + "TasksAccounting": "yes", + "TasksCurrent": "[not set]", + "TasksMax": "203813", + "TimeoutStartUSec": "1min 30s", + "TimeoutStopUSec": "1min 30s", + "TimerSlackNSec": "50000", + "Transient": "no", + "Type": "simple", + "UID": "[not set]", + "UMask": "0022", + "UnitFilePreset": "disabled", + "UnitFileState": "disabled", + "User": "root", + "UtmpMode": "init", + "Wants": "network.target", + "WatchdogTimestampMonotonic": "0", + "WatchdogUSec": "0" + } +} + +TASK [lock_manager : Debug VDI monitor setup] ********************************** +ok: [localhost] => {} + +MSG: + +VDI monitor service setup complete: + - Service: vdi-monitor.service + - Script: /usr/local/bin/vdi-monitor.sh + - Logs: /var/log/vdi-monitor.log, /var/log/ansible-vdi-reconfig.log + - Service enabled and started + + +TASK [lock_manager : Fail if lock file is invalid] ***************************** +skipping: [localhost] => { + "changed": false, + "false_condition": "lock_file_validation.rc != 0", + "skip_reason": "Conditional result was False" +} + +TASK [base_os : DEBUG About to import lock_manager for base_os] **************** +ok: [localhost] => {} + +MSG: + +About to import lock_manager for base_os + +TASK [base_os : Set current role for lock manager check] *********************** +ok: [localhost] => { + "ansible_facts": { + "current_role": "base_os" + }, + "changed": false +} + +TASK [lock_manager : Check if lock file exists] ******************************** +ok: [localhost] => { + "changed": false, + "stat": { + "atime": 1753370866.5072315, + "attr_flags": "", + "attributes": [], + "block_size": 4096, + "blocks": 8, + "charset": "us-ascii", + "checksum": "358a7f4095fbb49a0eedafd77e97040d1b9b5a26", + "ctime": 1753370866.2832148, + "dev": 2050, + "device_type": 0, + "executable": false, + "exists": true, + "gid": 0, + "gr_name": "root", + "inode": 891734, + "isblk": false, + "ischr": false, + "isdir": false, + "isfifo": false, + "isgid": false, + "islnk": false, + "isreg": true, + "issock": false, + "isuid": false, + "mimetype": "text/plain", + "mode": "0644", + "mtime": 1753370866.137204, + "nlink": 1, + "path": "/opt/vdi-setup/.vdi-lock.yaml", + "pw_name": "root", + "readable": true, + "rgrp": true, + "roth": true, + "rusr": true, + "size": 879, + "uid": 0, + "version": "3120075969", + "wgrp": false, + "woth": false, + "writeable": true, + "wusr": true, + "xgrp": false, + "xoth": false, + "xusr": false + } +} + +TASK [lock_manager : Debug lock file existence] ******************************** +ok: [localhost] => {} + +MSG: + +Lock file exists: True + +TASK [lock_manager : Load existing lock file] ********************************** +ok: [localhost] => { + "ansible_facts": { + "vdi_setup_status": { + "completed_roles": { + "base_os": { + "completed": false + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:27:44Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:27:44Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "ansible_included_var_files": [ + "/opt/vdi-setup/.vdi-lock.yaml" + ], + "changed": false +} + +TASK [lock_manager : Debug lock file load result] ****************************** +ok: [localhost] => {} + +MSG: + +Lock file loaded successfully: True + +TASK [lock_manager : Calculate current deployment hash] ************************ +ok: [localhost] => { + "ansible_facts": { + "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab" + }, + "changed": false +} + +TASK [lock_manager : Debug deployment hash calculation] ************************ +ok: [localhost] => {} + +MSG: + +Current deployment hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab +Deployment components: + - deployment_name: vdi-test-scott + - project_id: hpc-discovery-external + - vdi_tool: guacamole + - vnc_flavor: tigervnc + - user_provision: local_users + - vdi_webapp_port: 8081 + - vdi_resolution: 1920x1080 + - vdi_users count: 2 + + +TASK [lock_manager : Calculate current user secrets hash] ********************** +ok: [localhost] => { + "ansible_facts": { + "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" + }, + "changed": false +} + +TASK [lock_manager : Debug user secrets hash calculation] ********************** +ok: [localhost] => {} + +MSG: + +Current user secrets hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 +User configuration components: + - vdi_users (blueprint): [{"username": "alice", "port": 5901}, {"username": "bob", "port": 5902, "secret_name": "a-password-for-bob"}] + + +TASK [lock_manager : Set default values for missing lock file] ***************** +skipping: [localhost] => { + "changed": false, + "false_condition": "not lock_file_stat.stat.exists", + "skip_reason": "Conditional result was False" +} + +TASK [lock_manager : Debug lock file status] *********************************** +ok: [localhost] => {} + +MSG: + +Lock file status: + - exists: True + - deployment_hash: none + - user_secrets_hash: none + - force_rerun: False + + +TASK [lock_manager : Check if lock file exists] ******************************** +ok: [localhost] => { + "ansible_facts": { + "lock_file_exists": true + }, + "changed": false +} + +TASK [lock_manager : Check if current role is completed] *********************** +ok: [localhost] => { + "ansible_facts": { + "current_role_completed": false + }, + "changed": false +} + +TASK [lock_manager : Check if deployment hash matches] ************************* +ok: [localhost] => { + "ansible_facts": { + "deployment_hash_matches": false + }, + "changed": false +} + +TASK [lock_manager : Check if force rerun is enabled] ************************** +ok: [localhost] => { + "ansible_facts": { + "force_rerun_enabled": false + }, + "changed": false +} + +TASK [lock_manager : Determine if role should run] ***************************** +ok: [localhost] => { + "ansible_facts": { + "role_should_run": true + }, + "changed": false +} + +TASK [lock_manager : Debug role should run calculation] ************************ +ok: [localhost] => {} + +MSG: + +Role should run calculation for base_os: + - not lock_file_exists: False + - not current_role_completed: True + - not deployment_hash_matches: True + - force_rerun_enabled: False + - role_should_run: True + + +TASK [lock_manager : Debug role execution decision] **************************** +ok: [localhost] => {} + +MSG: + +Role execution decision for base_os: + - current_role: base_os + - ansible_role_name: lock_manager + - role_should_run: True + - lock_file_exists: True + - current_role_completed: False + - deployment_hash_matches: False + - force_rerun_enabled: False + + +TASK [lock_manager : Check if user configuration has changed] ****************** +ok: [localhost] => { + "ansible_facts": { + "user_config_changed": true + }, + "changed": false +} + +TASK [lock_manager : Set user secrets changed flag (will be updated by secret_manager role)] *** +ok: [localhost] => { + "ansible_facts": { + "user_secrets_changed": true + }, + "changed": false +} + +TASK [lock_manager : Debug user secrets change detection] ********************** +ok: [localhost] => {} + +MSG: + +User configuration change detection: + - user_config_changed: True + - user_secrets_changed: True + - stored_hash: none + - current_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 + + +TASK [lock_manager : Set lock check result] ************************************ +ok: [localhost] => { + "ansible_facts": { + "lock_check_result": { + "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "existing_deployment_hash": "none", + "existing_user_secrets_hash": "none", + "force_rerun": false, + "role_should_run": true, + "user_secrets_changed": true + } + }, + "changed": false +} + +TASK [lock_manager : Debug final lock check result] **************************** +ok: [localhost] => {} + +MSG: + +Final lock check result: + - role_should_run: True + - deployment_hash_changed: True + - user_secrets_changed: True + - force_rerun: False + + +TASK [base_os : DEBUG Finished import_role for base_os] ************************ +ok: [localhost] => {} + +MSG: + +Finished import_role for base_os + +TASK [base_os : Set role_should_run from lock check result] ******************** +ok: [localhost] => { + "ansible_facts": { + "role_should_run": true + }, + "changed": false +} + +TASK [base_os : Skip base_os tasks if role should not run] ********************* +skipping: [localhost] => { + "false_condition": "not role_should_run" +} + +TASK [base_os : Gather distribution facts] ************************************* +ok: [localhost] + +TASK [base_os : Check if NVIDIA GPU is present] ******************************** +changed: [localhost] => { + "changed": true, + "cmd": "lspci | grep -i nvidia", + "delta": "0:00:00.013393", + "end": "2025-07-24 15:27:55.866012", + "failed_when_result": false, + "rc": 1, + "start": "2025-07-24 15:27:55.852619" +} + +MSG: + +non-zero return code + +TASK [base_os : Include Ubuntu-specific tasks] ********************************* +skipping: [localhost] => { + "changed": false, + "false_condition": "ansible_facts.distribution == 'Ubuntu'", + "skip_reason": "Conditional result was False" +} + +TASK [base_os : Include Rocky Linux specific tasks] **************************** +included: /tmp/vdi/roles/base_os/tasks/rocky.yaml for localhost + +TASK [base_os : Set Rocky Linux specific variables] **************************** +ok: [localhost] => { + "ansible_facts": { + "gui_packages": [ + "@Xfce", + "@base-x" + ], + "vnc_pkg": "tigervnc-server", + "xorg_dev_packages": [ + "pkgconfig", + "xorg-x11-server-devel", + "xorg-x11-util-macros", + "libglvnd-devel" + ] + }, + "changed": false +} + +TASK [base_os : Update repository cache & install EPEL (Rocky Linux only)] ***** +ok: [localhost] => { + "changed": false, + "rc": 0, + "results": [] +} + +MSG: + +Nothing to do + +TASK [base_os : Install desktop prerequisites & VNC package (Rocky Linux)] ***** +changed: [localhost] => { + "changed": true, + "rc": 0, + "results": [ + "Group xfce-desktop installed.", + "Group base-x installed.", + "Installed: garcon-4.16.1-1.el8.x86_64", + "Installed: desktop-file-utils-0.26-1.el8.x86_64", + "Installed: libfontenc-devel-1.1.3-8.el8.x86_64", + "Installed: accountsservice-0.6.55-4.el8.x86_64", + "Installed: vte-profile-0.52.4-2.el8.x86_64", + "Installed: accountsservice-libs-0.6.55-4.el8.x86_64", + "Installed: vte291-0.52.4-2.el8.x86_64", + "Installed: bluez-5.63-5.el8_10.x86_64", + "Installed: bluez-libs-5.63-5.el8_10.x86_64", + "Installed: adwaita-cursor-theme-3.28.0-3.el8.noarch", + "Installed: bluez-obexd-5.63-5.el8_10.x86_64", + "Installed: adwaita-gtk2-theme-3.22.3-4.el8.x86_64", + "Installed: adwaita-icon-theme-3.28.0-3.el8.noarch", + "Installed: ostree-libs-2022.2-8.el8.x86_64", + "Installed: bolt-0.9.1-1.el8.x86_64", + "Installed: libasyncns-0.8-14.el8.x86_64", + "Installed: alsa-lib-1.2.10-2.el8.x86_64", + "Installed: python3-pysocks-1.6.8-3.el8.noarch", + "Installed: libavc1394-0.5.4-7.el8.x86_64", + "Installed: wavpack-5.1.0-16.el8.x86_64", + "Installed: thunar-archive-plugin-0.4.0-26.el8.x86_64", + "Installed: thunar-volman-4.16.0-3.el8.x86_64", + "Installed: poppler-20.11.0-12.el8_10.x86_64", + "Installed: poppler-data-0.4.9-2.el8.noarch", + "Installed: python3-requests-2.20.0-5.el8_10.noarch", + "Installed: xfce-polkit-0.3-3.el8.x86_64", + "Installed: xfce4-appfinder-4.16.1-3.el8.x86_64", + "Installed: poppler-glib-20.11.0-12.el8_10.x86_64", + "Installed: libquvi-0.9.4-12.el8.x86_64", + "Installed: libquvi-scripts-0.9.20131130-9.el8.noarch", + "Installed: hunspell-1.6.2-1.el8.x86_64", + "Installed: libraw1394-2.1.2-5.el8.x86_64", + "Installed: pangomm-2.40.1-6.el8.x86_64", + "Installed: webkit2gtk3-2.48.3-1.el8_10.x86_64", + "Installed: llvm-compat-libs-17.0.6-3.module+el8.10.0+1875+4f0b06db.x86_64", + "Installed: libical-3.0.3-3.el8.x86_64", + "Installed: xfce4-panel-4.16.3-1.el8.x86_64", + "Installed: xfce4-power-manager-4.16.0-1.el8.x86_64", + "Installed: xfce4-pulseaudio-plugin-0.4.3-3.el8.x86_64", + "Installed: bzip2-devel-1.0.6-28.el8_10.x86_64", + "Installed: sbc-1.3-9.el8.x86_64", + "Installed: xfce4-screensaver-4.16.0-3.el8.x86_64", + "Installed: pavucontrol-3.0-11.el8.x86_64", + "Installed: xfce4-session-4.16.0-4.el8.x86_64", + "Installed: xfce4-settings-4.16.5-2.el8.x86_64", + "Installed: gnome-themes-standard-3.22.3-4.el8.x86_64", + "Installed: xfce4-terminal-1.0.4-1.el8.x86_64", + "Installed: hunspell-en-0.20140811.1-12.el8.noarch", + "Installed: hunspell-en-GB-0.20140811.1-12.el8.noarch", + "Installed: hunspell-en-US-0.20140811.1-12.el8.noarch", + "Installed: mobile-broadband-provider-info-20210805-1.el8.noarch", + "Installed: xfconf-4.16.0-1.el8.x86_64", + "Installed: xfdesktop-4.16.0-3.el8.x86_64", + "Installed: webkit2gtk3-jsc-2.48.3-1.el8_10.x86_64", + "Installed: xfwm4-4.16.1-1.el8.x86_64", + "Installed: libcanberra-0.30-18.el8.x86_64", + "Installed: webrtc-audio-processing-0.3-10.el8.x86_64", + "Installed: python3-urllib3-1.24.2-8.el8_10.noarch", + "Installed: libmousepad0-0.5.6-1.el8.x86_64", + "Installed: libcanberra-gtk3-0.30-18.el8.x86_64", + "Installed: crda-3.18_2020.04.29-1.el8.noarch", + "Installed: cairo-gobject-1.15.12-6.el8.x86_64", + "Installed: tumbler-0.2.7-1.el8.x86_64", + "Installed: woff2-1.0.2-5.el8.x86_64", + "Installed: cairomm-1.12.0-8.el8.x86_64", + "Installed: libmodman-2.0.1-17.el8.x86_64", + "Installed: libXvMC-1.0.12-1.el8.x86_64", + "Installed: aspell-12:0.60.6.1-22.el8.x86_64", + "Installed: libdbusmenu-16.04.0-12.el8.x86_64", + "Installed: libdbusmenu-gtk3-16.04.0-12.el8.x86_64", + "Installed: xcb-util-0.4.0-10.el8.x86_64", + "Installed: python3-chardet-3.0.4-7.el8.noarch", + "Installed: lua-expat-1.3.0-12.el8.1.x86_64", + "Installed: gsettings-desktop-schemas-3.32.0-6.el8.x86_64", + "Installed: libdmx-1.1.4-3.el8.x86_64", + "Installed: rocky-logos-86.3-1.el8.x86_64", + "Installed: lua-socket-3.0-0.17.rc1.el8.x86_64", + "Installed: libdrm-2.4.115-2.el8.x86_64", + "Installed: xdg-desktop-portal-1.12.6-1.el8.x86_64", + "Installed: libdrm-devel-2.4.115-2.el8.x86_64", + "Installed: cheese-libs-2:3.28.0-4.el8_6.x86_64", + "Installed: libdv-1.0.0-27.el8.x86_64", + "Installed: libshout-2.2.2-19.el8.x86_64", + "Installed: libdvdnav-5.0.3-8.el8.x86_64", + "Installed: libsigc++20-2.10.0-6.el8.x86_64", + "Installed: libdvdread-5.0.3-9.el8.x86_64", + "Installed: xdg-desktop-portal-gtk-1.12.0-1.el8.x86_64", + "Installed: samba-common-4.19.4-9.el8_10.noarch", + "Installed: libepoxy-1.5.8-1.el8.x86_64", + "Installed: python3-pycurl-7.43.0.2-4.el8.x86_64", + "Installed: xorg-x11-drv-ati-19.1.0-1.el8.x86_64", + "Installed: dejavu-fonts-common-2.35-7.el8.noarch", + "Installed: libsndfile-1.0.28-16.el8_10.x86_64", + "Installed: xorg-x11-drv-evdev-2.10.6-2.el8.x86_64", + "Installed: dejavu-sans-mono-fonts-2.35-7.el8.noarch", + "Installed: xorg-x11-drv-fbdev-0.5.0-2.el8.x86_64", + "Installed: flac-libs-1.3.2-11.el8.x86_64", + "Installed: libevdev-1.10.0-1.el8.x86_64", + "Installed: nm-connection-editor-1.26.0-1.el8.x86_64", + "Installed: flatpak-1.12.9-3.el8_10.x86_64", + "Installed: libexif-0.6.22-5.el8_3.x86_64", + "Installed: xorg-x11-drv-intel-2.99.917-41.20210115.el8.x86_64", + "Installed: xorg-x11-drv-libinput-0.29.0-1.el8.x86_64", + "Installed: xorg-x11-drv-nouveau-1:1.0.15-4.el8.1.x86_64", + "Installed: xorg-x11-drv-qxl-0.1.5-11.el8.x86_64", + "Installed: xorg-x11-drv-vesa-2.4.0-3.el8.x86_64", + "Installed: xorg-x11-drv-vmware-13.2.1-8.el8.x86_64", + "Installed: xorg-x11-drv-wacom-0.38.0-1.el8.x86_64", + "Installed: xorg-x11-drv-wacom-serial-support-0.38.0-1.el8.x86_64", + "Installed: flatpak-selinux-1.12.9-3.el8_10.noarch", + "Installed: libsrtp-1.5.4-8.el8.x86_64", + "Installed: sound-theme-freedesktop-0.8-9.el8.noarch", + "Installed: flatpak-session-helper-1.12.9-3.el8_10.x86_64", + "Installed: soundtouch-2.0.0-3.el8.x86_64", + "Installed: iw-5.19-1.el8.1.x86_64", + "Installed: libpciaccess-0.14-1.el8.x86_64", + "Installed: libgdata-0.17.9-4.el8.x86_64", + "Installed: at-spi2-atk-2.26.2-1.el8.x86_64", + "Installed: libpng-devel-2:1.6.34-5.el8.x86_64", + "Installed: at-spi2-core-2.28.0-1.el8.x86_64", + "Installed: xorg-x11-proto-devel-2020.1-3.el8.noarch", + "Installed: libproxy-0.4.15-5.5.el8_10.x86_64", + "Installed: hyphen-2.8.8-9.el8.x86_64", + "Installed: speex-1.2.0-1.el8.x86_64", + "Installed: libglvnd-1:1.3.4-2.el8.x86_64", + "Installed: speexdsp-1.2-0.13.rc3.el8.x86_64", + "Installed: libglvnd-core-devel-1:1.3.4-2.el8.x86_64", + "Installed: samba-client-libs-4.19.4-9.el8_10.x86_64", + "Installed: atkmm-2.24.2-7.el8.x86_64", + "Installed: libglvnd-devel-1:1.3.4-2.el8.x86_64", + "Installed: libglvnd-egl-1:1.3.4-2.el8.x86_64", + "Installed: libglvnd-gles-1:1.3.4-2.el8.x86_64", + "Installed: clutter-1.26.2-8.el8.x86_64", + "Installed: libglvnd-glx-1:1.3.4-2.el8.x86_64", + "Installed: libtheora-1:1.1.1-21.el8.x86_64", + "Installed: libglvnd-opengl-1:1.3.4-2.el8.x86_64", + "Installed: clutter-gst3-3.0.26-1.el8.x86_64", + "Installed: libgnomekbd-3.26.0-4.el8.x86_64", + "Installed: clutter-gtk-1.8.4-3.el8.x86_64", + "Installed: tigervnc-license-1.15.0-7.el8_10.noarch", + "Installed: xorg-x11-server-Xorg-1.20.11-26.el8_10.x86_64", + "Installed: gcr-3.28.0-1.el8.x86_64", + "Installed: xorg-x11-server-Xwayland-21.1.3-18.el8_10.x86_64", + "Installed: tigervnc-selinux-1.15.0-7.el8_10.noarch", + "Installed: xorg-x11-server-common-1.20.11-26.el8_10.x86_64", + "Installed: p11-kit-server-0.23.22-2.el8.x86_64", + "Installed: xorg-x11-utils-7.5-28.el8.x86_64", + "Installed: xorg-x11-xauth-1:1.0.9-12.el8.x86_64", + "Installed: samba-common-libs-4.19.4-9.el8_10.x86_64", + "Installed: xorg-x11-xinit-1.3.4-18.el8.x86_64", + "Installed: xorg-x11-xinit-session-1.3.4-18.el8.x86_64", + "Installed: xorg-x11-xkb-utils-7.7-28.el8.x86_64", + "Installed: tigervnc-server-1.15.0-7.el8_10.x86_64", + "Installed: libgtop2-2.38.0-3.el8.x86_64", + "Installed: gdm-1:40.0-27.el8.x86_64", + "Installed: tigervnc-server-minimal-1.15.0-7.el8_10.x86_64", + "Installed: startup-notification-0.12-15.el8.x86_64", + "Installed: cogl-1.22.2-10.el8.x86_64", + "Installed: color-filesystem-1-20.el8.noarch", + "Installed: colord-1.4.2-1.el8.x86_64", + "Installed: libusbmuxd-1.0.10-9.el8.x86_64", + "Installed: colord-gtk-0.1.26-8.el8.x86_64", + "Installed: colord-libs-1.4.2-1.el8.x86_64", + "Installed: libv4l-1.14.2-3.el8.x86_64", + "Installed: keybinder3-0.3.2-4.el8.x86_64", + "Installed: zenity-3.28.1-2.el8.x86_64", + "Installed: ibus-1.5.19-15.el8_10.x86_64", + "Installed: geoclue2-2.5.5-2.el8.x86_64", + "Installed: ibus-gtk2-1.5.19-15.el8_10.x86_64", + "Installed: geoclue2-libs-2.5.5-2.el8.x86_64", + "Installed: ibus-gtk3-1.5.19-15.el8_10.x86_64", + "Installed: geocode-glib-3.26.0-4.el8_10.x86_64", + "Installed: ibus-libs-1.5.19-15.el8_10.x86_64", + "Installed: lame-libs-3.100-6.el8.x86_64", + "Installed: libgweather-3.28.2-4.el8.x86_64", + "Installed: ibus-setup-1.5.19-15.el8_10.noarch", + "Installed: libsmbclient-4.19.4-9.el8_10.x86_64", + "Installed: mousepad-0.5.6-1.el8.x86_64", + "Installed: switcheroo-control-1.1-5.el8.x86_64", + "Installed: mesa-dri-drivers-23.1.4-4.el8_10.x86_64", + "Installed: libiec61883-1.2.0-18.el8.x86_64", + "Installed: mesa-filesystem-23.1.4-4.el8_10.x86_64", + "Installed: libsoup-2.62.3-9.el8_10.x86_64", + "Installed: mesa-libEGL-23.1.4-4.el8_10.x86_64", + "Installed: libimobiledevice-1.2.0-16.el8.x86_64", + "Installed: system-config-printer-libs-1.5.11-13.el8.noarch", + "Installed: libinput-1.16.3-3.el8_6.x86_64", + "Installed: mesa-libGL-23.1.4-4.el8_10.x86_64", + "Installed: freetype-devel-2.9.1-10.el8_10.x86_64", + "Installed: mesa-libGL-devel-23.1.4-4.el8_10.x86_64", + "Installed: enchant2-2.2.3-3.el8.x86_64", + "Installed: taglib-1.11.1-8.el8.x86_64", + "Installed: fuse-2.9.7-19.el8.x86_64", + "Installed: pulseaudio-14.0-4.el8.x86_64", + "Installed: iio-sensor-proxy-2.4-3.el8.x86_64", + "Installed: pulseaudio-libs-14.0-4.el8.x86_64", + "Installed: rest-0.8.1-2.el8.x86_64", + "Installed: pulseaudio-libs-glib2-14.0-4.el8.x86_64", + "Installed: pulseaudio-module-bluetooth-14.0-4.el8.x86_64", + "Installed: libappstream-glib-0.7.14-3.el8.x86_64", + "Installed: grilo-0.3.6-3.el8.x86_64", + "Installed: pinentry-gtk-1.1.0-2.el8.x86_64", + "Installed: totem-pl-parser-3.26.1-2.el8.x86_64", + "Installed: mesa-libgbm-23.1.4-4.el8_10.x86_64", + "Installed: pipewire-0.3.6-1.el8.x86_64", + "Installed: evolution-data-server-3.28.5-24.el8.x86_64", + "Installed: libXfont2-devel-2.0.3-2.el8.x86_64", + "Installed: evolution-data-server-langpacks-3.28.5-24.el8.noarch", + "Installed: gsm-1.0.17-5.el8.x86_64", + "Installed: evolution-data-server-ui-3.28.5-24.el8.x86_64", + "Installed: mesa-libglapi-23.1.4-4.el8_10.x86_64", + "Installed: pipewire-libs-0.3.6-1.el8.x86_64", + "Installed: rocky-backgrounds-86.3-1.el8.noarch", + "Installed: mesa-libxatracker-23.1.4-4.el8_10.x86_64", + "Installed: pixman-devel-0.38.4-4.el8.x86_64", + "Installed: gstreamer1-1.16.1-2.el8.x86_64", + "Installed: libstemmer-0-10.585svn.el8.x86_64", + "Installed: libvisual-1:0.4.0-25.el8.x86_64", + "Installed: gstreamer1-plugins-bad-free-1.16.1-5.el8_10.x86_64", + "Installed: Thunar-4.16.8-1.el8.x86_64", + "Installed: libxfce4ui-4.16.0-2.el8.x86_64", + "Installed: gstreamer1-plugins-base-1.16.1-5.el8_10.x86_64", + "Installed: exo-4.16.2-1.el8.x86_64", + "Installed: libxfce4util-4.16.0-4.el8.x86_64", + "Installed: twolame-libs-0.3.13-12.el8.x86_64", + "Installed: libvorbis-1:1.3.6-2.el8.x86_64", + "Installed: gjs-1.56.2-6.el8.x86_64", + "Installed: glib-networking-2.56.1-1.1.el8.x86_64", + "Installed: gstreamer1-plugins-good-1.16.1-5.el8_10.x86_64", + "Installed: libvpx-1.7.0-12.el8_10.x86_64", + "Installed: libwacom-1.6-3.el8.x86_64", + "Installed: libwacom-data-1.6-3.el8.noarch", + "Installed: ModemManager-glib-1.20.2-1.el8.x86_64", + "Installed: libwayland-client-1.21.0-1.el8.x86_64", + "Installed: libwayland-cursor-1.21.0-1.el8.x86_64", + "Installed: libwayland-egl-1.21.0-1.el8.x86_64", + "Installed: libwayland-server-1.21.0-1.el8.x86_64", + "Installed: plymouth-graphics-libs-0.9.4-11.20200615git1e36e30.el8.x86_64", + "Installed: glibmm24-2.56.0-2.el8.x86_64", + "Installed: gtk3-3.22.30-12.el8_10.x86_64", + "Installed: plymouth-plugin-label-0.9.4-11.20200615git1e36e30.el8.x86_64", + "Installed: glx-utils-8.4.0-5.20181118git1830dcb.el8.x86_64", + "Installed: xorg-x11-server-devel-1.20.11-26.el8_10.x86_64", + "Installed: plymouth-plugin-two-step-0.9.4-11.20200615git1e36e30.el8.x86_64", + "Installed: xorg-x11-util-macros-1.19.2-1.el8.noarch", + "Installed: libX11-devel-1.6.8-9.el8_10.x86_64", + "Installed: plymouth-system-theme-0.9.4-11.20200615git1e36e30.el8.x86_64", + "Installed: gnome-bluetooth-1:3.34.3-1.el8.x86_64", + "Installed: gtkmm30-3.22.2-3.el8.x86_64", + "Installed: gnome-bluetooth-libs-1:3.34.3-1.el8.x86_64", + "Installed: gtksourceview3-3.24.9-1.el8.x86_64", + "Installed: libX11-xcb-1.6.8-9.el8_10.x86_64", + "Installed: libwnck3-3.24.1-2.el8.x86_64", + "Installed: libXScrnSaver-1.2.3-1.el8.x86_64", + "Installed: plymouth-theme-charge-0.9.4-11.20200615git1e36e30.el8.x86_64", + "Installed: plymouth-theme-spinner-0.9.4-11.20200615git1e36e30.el8.x86_64", + "Installed: rtkit-0.11-19.el8.x86_64", + "Installed: NetworkManager-wifi-1:1.40.16-19.el8_10.x86_64", + "Installed: gnome-control-center-3.28.2-37.el8.x86_64", + "Installed: gnome-control-center-filesystem-3.28.2-37.el8.noarch", + "Installed: libXau-devel-1.0.9-3.el8.x86_64", + "Installed: gnome-desktop3-3.32.2-3.el8.x86_64", + "Installed: libnma-1.8.38-1.el8.x86_64", + "Installed: libnotify-0.7.7-6.el8.x86_64", + "Installed: libxcb-devel-1.13.1-1.el8.x86_64", + "Installed: upower-0.99.7-4.el8_7.x86_64", + "Installed: liboauth-1.0.3-9.el8.x86_64", + "Installed: libxkbcommon-x11-0.9.1-1.el8.x86_64", + "Installed: libxkbfile-1.1.0-1.el8.x86_64", + "Installed: libogg-2:1.3.2-10.el8.x86_64", + "Installed: libxklavier-5.4-11.el8.x86_64", + "Installed: wpa_supplicant-1:2.10-2.el8.x86_64", + "Installed: libXdmcp-1.1.3-1.el8.x86_64", + "Installed: gnome-keyring-3.28.2-2.el8_10.x86_64", + "Installed: gnome-keyring-pam-3.28.2-2.el8_10.x86_64", + "Installed: python3-idna-2.5-7.el8_10.noarch", + "Installed: gnome-menus-3.13.3-12.el8.x86_64", + "Installed: GConf2-3.2.6-22.el8.x86_64", + "Installed: gnome-online-accounts-3.28.2-7.el8.x86_64", + "Installed: libXfont2-2.0.3-2.el8.x86_64", + "Installed: libxshmfence-1.3-2.el8.x86_64", + "Installed: mpg123-libs-1.32.9-1.el8_10.x86_64", + "Installed: gnome-session-3.28.1-21.el8.x86_64", + "Installed: python3-cairo-1.16.3-6.el8.x86_64", + "Installed: gnome-session-wayland-session-3.28.1-21.el8.x86_64", + "Installed: gnome-session-xsession-3.28.1-21.el8.x86_64", + "Installed: gnome-settings-daemon-3.32.0-20.el8.x86_64", + "Installed: libwbclient-4.19.4-9.el8_10.x86_64", + "Installed: gnome-shell-3.32.2-57.el8_10.x86_64", + "Installed: iso-codes-3.79-2.el8.noarch", + "Installed: libpciaccess-devel-0.14-1.el8.x86_64", + "Installed: mtdev-1.1.5-12.el8.x86_64", + "Installed: cups-pk-helper-0.2.6-5.el8.x86_64", + "Installed: openssh-askpass-8.0p1-25.el8_10.x86_64", + "Installed: mutter-3.32.2-73.el8_10.x86_64", + "Installed: python3-cups-1.9.72-21.el8.rocky.x86_64", + "Installed: libXres-1.2.0-4.el8.x86_64", + "Installed: libplist-2.0.0-10.el8.x86_64", + "Installed: opus-1.3-0.4.beta.el8.x86_64", + "Installed: avahi-glib-0.7-27.el8_10.1.x86_64", + "Installed: vino-3.22.0-11.el8.x86_64", + "Installed: harfbuzz-icu-1.7.5-4.el8.x86_64", + "Installed: libXtst-1.2.3-7.el8.x86_64", + "Installed: orc-0.4.28-4.el8_10.x86_64", + "Installed: libXv-1.0.11-7.el8.x86_64", + "Installed: abattis-cantarell-fonts-0.0.25-6.el8.noarch", + "Installed: dbus-x11-1:1.12.8-26.el8.x86_64", + "Installed: dconf-0.28.0-4.el8.x86_64", + "Installed: libXxf86dga-1.1.5-1.el8.x86_64", + "Installed: python3-gobject-3.28.3-2.el8.x86_64" + ] +} + +TASK [base_os : Include Debian-specific tasks] ********************************* +skipping: [localhost] => { + "changed": false, + "false_condition": "ansible_facts.distribution == 'Debian'", + "skip_reason": "Conditional result was False" +} + +TASK [base_os : Ensure requests package is installed] ************************** +ok: [localhost] => { + "changed": false, + "cmd": [ + "/usr/local/ghpc-venv/bin/python3", + "-m", + "pip.__main__", + "install", + "requests" + ], + "name": [ + "requests" + ], + "requirements": null, + "state": "present", + "version": null, + "virtualenv": null +} + +STDOUT: + +Requirement already satisfied: requests in /usr/local/ghpc-venv/lib64/python3.12/site-packages (2.32.4) +Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/ghpc-venv/lib64/python3.12/site-packages (from requests) (3.4.2) +Requirement already satisfied: idna<4,>=2.5 in /usr/local/ghpc-venv/lib64/python3.12/site-packages (from requests) (3.10) +Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/ghpc-venv/lib64/python3.12/site-packages (from requests) (2.5.0) +Requirement already satisfied: certifi>=2017.4.17 in /usr/local/ghpc-venv/lib64/python3.12/site-packages (from requests) (2025.7.14) + + +TASK [base_os : Ensure Python & tooling are installed] ************************* +ok: [localhost] => { + "changed": false, + "rc": 0, + "results": [] +} + +MSG: + +Nothing to do + +TASK [base_os : Provision GPU setup] ******************************************* +skipping: [localhost] => { + "changed": false, + "false_condition": "nvidia_gpu_present.rc == 0", + "skip_reason": "Conditional result was False" +} + +TASK [base_os : Set current role for lock manager completion] ****************** +ok: [localhost] => { + "ansible_facts": { + "current_role": "base_os", + "role_completed": true + }, + "changed": false +} + +TASK [lock_manager : Debug create lock operation] ****************************** +ok: [localhost] => {} + +MSG: + +Creating/updating lock file for role: base_os + +TASK [lock_manager : Check if lock file exists] ******************************** +ok: [localhost] => { + "changed": false, + "stat": { + "atime": 1753370866.5072315, + "attr_flags": "", + "attributes": [], + "block_size": 4096, + "blocks": 8, + "charset": "us-ascii", + "checksum": "358a7f4095fbb49a0eedafd77e97040d1b9b5a26", + "ctime": 1753370866.2832148, + "dev": 2050, + "device_type": 0, + "executable": false, + "exists": true, + "gid": 0, + "gr_name": "root", + "inode": 891734, + "isblk": false, + "ischr": false, + "isdir": false, + "isfifo": false, + "isgid": false, + "islnk": false, + "isreg": true, + "issock": false, + "isuid": false, + "mimetype": "text/plain", + "mode": "0644", + "mtime": 1753370866.137204, + "nlink": 1, + "path": "/opt/vdi-setup/.vdi-lock.yaml", + "pw_name": "root", + "readable": true, + "rgrp": true, + "roth": true, + "rusr": true, + "size": 879, + "uid": 0, + "version": "3120075969", + "wgrp": false, + "woth": false, + "writeable": true, + "wusr": true, + "xgrp": false, + "xoth": false, + "xusr": false + } +} + +TASK [lock_manager : Debug lock file existence] ******************************** +ok: [localhost] => {} + +MSG: + +Lock file exists: True + +TASK [lock_manager : Create initial lock file structure] *********************** +skipping: [localhost] => { + "changed": false, + "false_condition": "not lock_file_stat.stat.exists", + "skip_reason": "Conditional result was False" +} + +TASK [lock_manager : Debug lock file creation] ********************************* +skipping: [localhost] => { + "false_condition": "debug | default(false) and not lock_file_stat.stat.exists" +} + +TASK [lock_manager : Load existing lock file data] ***************************** +ok: [localhost] => { + "ansible_facts": { + "vdi_setup_status": { + "completed_roles": { + "base_os": { + "completed": false + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:27:44Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:27:44Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "ansible_included_var_files": [ + "/opt/vdi-setup/.vdi-lock.yaml" + ], + "changed": false +} + +TASK [lock_manager : Get current lock data] ************************************ +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": false + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:27:44Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:27:44Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Update deployment hash] *********************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": false + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:27:44Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:27:44Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Get current timestamp] ************************************ +changed: [localhost] => { + "changed": true, + "cmd": [ + "date", + "-u", + "+%Y-%m-%dT%H:%M:%SZ" + ], + "delta": "0:00:00.002192", + "end": "2025-07-24 15:29:50.438490", + "rc": 0, + "start": "2025-07-24 15:29:50.436298" +} + +STDOUT: + +2025-07-24T15:29:50Z + +TASK [lock_manager : Update last updated timestamp] **************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": false + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:29:50Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:27:44Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Get current user secrets status] ************************** +ok: [localhost] => { + "ansible_facts": { + "current_user_secrets": { + "last_secret_check": "2025-07-24T15:27:44Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Create user secrets update] ******************************* +ok: [localhost] => { + "ansible_facts": { + "user_secrets_update": { + "last_secret_check": "2025-07-24T15:29:50Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" + } + }, + "changed": false +} + +TASK [lock_manager : Update user secrets hash] ********************************* +ok: [localhost] => { + "ansible_facts": { + "updated_user_secrets": { + "last_secret_check": "2025-07-24T15:29:50Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Apply user secrets update] ******************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": false + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:29:50Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:29:50Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Create role status entry] ********************************* +ok: [localhost] => { + "ansible_facts": { + "role_status_entry": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + } + }, + "changed": false +} + +TASK [lock_manager : Debug role status entry] ********************************** +ok: [localhost] => {} + +MSG: + +Role status entry for base_os: + - completed: True + - completed_at: 2025-07-24T15:29:50Z + + +TASK [lock_manager : Debug current lock data before role update] *************** +ok: [localhost] => {} + +MSG: + +Current lock data before updating base_os: + - current_lock_data: completed_roles: + base_os: + completed: false + lock_manager: + completed: true + completed_at: '2025-07-24T15:27:44Z' + secret_manager: + completed: false + user_provision: + completed: false + vdi_tool: + completed: false + vnc: + completed: false +created_at: '2025-07-24T15:27:41Z' +deployment_hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab +deployment_name: vdi-test-scott +force_rerun: false +last_updated: '2025-07-24T15:29:50Z' +lock_version: '1.0' +setup_status: configuring +user_management: + current_users: + - alice + - bob + previous_users: [] + removed_users: [] +user_secrets_status: + last_secret_check: '2025-07-24T15:29:50Z' + user_secrets_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 + users_updated: + - alice + - bob + + - current_completed_roles: base_os: + completed: false +lock_manager: + completed: true + completed_at: '2025-07-24T15:27:44Z' +secret_manager: + completed: false +user_provision: + completed: false +vdi_tool: + completed: false +vnc: + completed: false + + - role_completed: True + - current_role: base_os + + +TASK [lock_manager : Get current completed roles] ****************************** +ok: [localhost] => { + "ansible_facts": { + "current_completed_roles": { + "base_os": { + "completed": false + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + } + }, + "changed": false +} + +TASK [lock_manager : Update completed roles] *********************************** +ok: [localhost] => { + "ansible_facts": { + "updated_completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + } + }, + "changed": false +} + +TASK [lock_manager : Update lock data with completed roles] ******************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:29:50Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:29:50Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Debug updated lock data after role update] **************** +ok: [localhost] => {} + +MSG: + +Updated lock data after updating base_os: + - current_role: base_os + - role_completed: True + - completed_roles: {'base_os': {'completed': True, 'completed_at': '2025-07-24T15:29:50Z'}, 'lock_manager': {'completed': True, 'completed_at': '2025-07-24T15:27:44Z'}, 'secret_manager': {'completed': False}, 'user_provision': {'completed': False}, 'vdi_tool': {'completed': False}, 'vnc': {'completed': False}} + + +TASK [lock_manager : Get current users updated list] *************************** +ok: [localhost] => { + "ansible_facts": { + "current_users_updated": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Get new usernames list] *********************************** +ok: [localhost] => { + "ansible_facts": { + "new_usernames": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Combine users updated lists] ****************************** +ok: [localhost] => { + "ansible_facts": { + "combined_users_updated": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Debug user deduplication] ********************************* +ok: [localhost] => {} + +MSG: + +User deduplication for base_os: + - current_users_updated: ['alice', 'bob'] + - new_usernames: ['alice', 'bob'] + - combined_users_updated: ['alice', 'bob'] + + +TASK [lock_manager : Get current user secrets for list update] ***************** +ok: [localhost] => { + "ansible_facts": { + "current_user_secrets_for_list": { + "last_secret_check": "2025-07-24T15:29:50Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Update users updated list] ******************************** +ok: [localhost] => { + "ansible_facts": { + "updated_user_secrets_with_list": { + "last_secret_check": "2025-07-24T15:29:50Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Apply users updated list] ********************************* +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:29:50Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:29:50Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Get current user management data] ************************* +ok: [localhost] => { + "ansible_facts": { + "current_user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [], + "removed_users": [] + } + }, + "changed": false +} + +TASK [lock_manager : Get current usernames list] ******************************* +ok: [localhost] => { + "ansible_facts": { + "current_usernames": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Update user management data] ****************************** +ok: [localhost] => { + "ansible_facts": { + "updated_user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + } + }, + "changed": false +} + +TASK [lock_manager : Apply user management update] ***************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:29:50Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:29:50Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Debug user management] ************************************ +ok: [localhost] => {} + +MSG: + +User management for base_os: + - current_users: ['alice', 'bob'] + - previous_users: ['alice', 'bob'] + - removed_users: [] + + +TASK [lock_manager : Check if all roles are completed] ************************* +ok: [localhost] => { + "ansible_facts": { + "all_roles_completed": false + }, + "changed": false +} + +TASK [lock_manager : Update setup status based on completion] ****************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:29:50Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:29:50Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Debug setup status update] ******************************** +ok: [localhost] => {} + +MSG: + +Setup status update: + - all_roles_completed: False + - setup_status: configuring + - completed_roles_count: 2 + - total_roles_count: 6 + + +TASK [lock_manager : Write updated lock file] ********************************** +changed: [localhost] => { + "changed": true, + "checksum": "9dd77a3585697aef08aed09e819bb4819f75aeb3", + "dest": "/opt/vdi-setup/.vdi-lock.yaml", + "gid": 0, + "group": "root", + "md5sum": "31b80e47ec3641c2918f8ff0403d2956", + "mode": "0644", + "owner": "root", + "size": 940, + "src": "/root/.ansible/tmp/ansible-tmp-1753370991.526421-18369-193730364017589/source", + "state": "file", + "uid": 0 +} + +TASK [lock_manager : Debug VM metadata update] ********************************* +ok: [localhost] => {} + +MSG: + +VM metadata update info: + - hostname: vdi-test-scott-rocky-0 + - zone: us-central1-a + - lock_file_changed: True + + +TASK [lock_manager : Get lock file content as base64] ************************** +changed: [localhost] => { + "changed": true, + "cmd": "cat /opt/vdi-setup/.vdi-lock.yaml | base64 -w 0", + "delta": "0:00:00.003801", + "end": "2025-07-24 15:29:52.075433", + "rc": 0, + "start": "2025-07-24 15:29:52.071632" +} + +STDOUT: + +dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IGZhbHNlCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiBmYWxzZQogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogZmFsc2UKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiBmYWxzZQogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogY29uZmlndXJpbmcKICB1c2VyX21hbmFnZW1lbnQ6CiAgICBjdXJyZW50X3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHByZXZpb3VzX3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHJlbW92ZWRfdXNlcnM6IFtdCiAgdXNlcl9zZWNyZXRzX3N0YXR1czoKICAgIGxhc3Rfc2VjcmV0X2NoZWNrOiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICB1c2VyX3NlY3JldHNfaGFzaDogNmE2YTAyZDNkMDk5MzNjYTI4MmFlMmIwMjMzNjM0M2M5ZDczNzAxZTQ1YmQ4MTQ3NWZiZDkyZGNmOWQyZjNiNgogICAgdXNlcnNfdXBkYXRlZDoKICAgIC0gYWxpY2UKICAgIC0gYm9iCg== + +TASK [lock_manager : Add VDI lock file content to VM metadata] ***************** +changed: [localhost] => { + "changed": true, + "cmd": [ + "gcloud", + "compute", + "instances", + "add-metadata", + "vdi-test-scott-rocky-0", + "--metadata", + "vdi-lock-content=dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IGZhbHNlCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiBmYWxzZQogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogZmFsc2UKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiBmYWxzZQogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogY29uZmlndXJpbmcKICB1c2VyX21hbmFnZW1lbnQ6CiAgICBjdXJyZW50X3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHByZXZpb3VzX3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHJlbW92ZWRfdXNlcnM6IFtdCiAgdXNlcl9zZWNyZXRzX3N0YXR1czoKICAgIGxhc3Rfc2VjcmV0X2NoZWNrOiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICB1c2VyX3NlY3JldHNfaGFzaDogNmE2YTAyZDNkMDk5MzNjYTI4MmFlMmIwMjMzNjM0M2M5ZDczNzAxZTQ1YmQ4MTQ3NWZiZDkyZGNmOWQyZjNiNgogICAgdXNlcnNfdXBkYXRlZDoKICAgIC0gYWxpY2UKICAgIC0gYm9iCg==", + "--zone=us-central1-a" + ], + "delta": "0:00:04.336413", + "end": "2025-07-24 15:29:56.603337", + "failed_when_result": false, + "rc": 0, + "start": "2025-07-24 15:29:52.266924" +} + +STDERR: + +Updated [https://www.googleapis.com/compute/v1/projects/hpc-discovery-external/zones/us-central1-a/instances/vdi-test-scott-rocky-0]. + +TASK [lock_manager : Debug lock file write] ************************************ +ok: [localhost] => {} + +MSG: + +Lock file written: True + +TASK [lock_manager : Verify lock file integrity] ******************************* +ok: [localhost] => { + "changed": false, + "cmd": [ + "python3", + "-c", + "import yaml; yaml.safe_load(open('/opt/vdi-setup/.vdi-lock.yaml'))" + ], + "delta": "0:00:00.039517", + "end": "2025-07-24 15:29:56.877395", + "rc": 0, + "start": "2025-07-24 15:29:56.837878" +} + +TASK [lock_manager : Debug lock file validation] ******************************* +ok: [localhost] => {} + +MSG: + +Lock file validation: True + +TASK [lock_manager : Check if VDI monitor service exists] ********************** +ok: [localhost] => { + "changed": false, + "stat": { + "atime": 1753370874.1548011, + "attr_flags": "", + "attributes": [], + "block_size": 4096, + "blocks": 8, + "charset": "us-ascii", + "checksum": "f6de22707f8d9248014f679a2f1930451e2bf635", + "ctime": 1753370872.6256874, + "dev": 2050, + "device_type": 0, + "executable": false, + "exists": true, + "gid": 0, + "gr_name": "root", + "inode": 285214140, + "isblk": false, + "ischr": false, + "isdir": false, + "isfifo": false, + "isgid": false, + "islnk": false, + "isreg": true, + "issock": false, + "isuid": false, + "mimetype": "text/plain", + "mode": "0644", + "mtime": 1753370872.4766762, + "nlink": 1, + "path": "/etc/systemd/system/vdi-monitor.service", + "pw_name": "root", + "readable": true, + "rgrp": true, + "roth": true, + "rusr": true, + "size": 283, + "uid": 0, + "version": "1338188335", + "wgrp": false, + "woth": false, + "writeable": true, + "wusr": true, + "xgrp": false, + "xoth": false, + "xusr": false + } +} + +TASK [lock_manager : Debug VDI monitor setup conditions] *********************** +ok: [localhost] => {} + +MSG: + +VDI monitor setup conditions: + - current_role: base_os + - vdi_monitor_service_exists: True + - lock_file_creation.changed: False + - will_setup_monitor: False + + +TASK [lock_manager : Setup VDI monitoring service (first time only)] *********** +skipping: [localhost] => { + "changed": false, + "false_condition": "not vdi_monitor_service_stat.stat.exists", + "skip_reason": "Conditional result was False" +} + +TASK [lock_manager : Fail if lock file is invalid] ***************************** +skipping: [localhost] => { + "changed": false, + "false_condition": "lock_file_validation.rc != 0", + "skip_reason": "Conditional result was False" +} + +TASK [secret_manager : Set current role for lock manager check] **************** +ok: [localhost] => { + "ansible_facts": { + "current_role": "secret_manager" + }, + "changed": false +} + +TASK [lock_manager : Check if lock file exists] ******************************** +ok: [localhost] => { + "changed": false, + "stat": { + "atime": 1753370992.073591, + "attr_flags": "", + "attributes": [], + "block_size": 4096, + "blocks": 8, + "charset": "us-ascii", + "checksum": "9dd77a3585697aef08aed09e819bb4819f75aeb3", + "ctime": 1753370991.845574, + "dev": 2050, + "device_type": 0, + "executable": false, + "exists": true, + "gid": 0, + "gr_name": "root", + "inode": 185115083, + "isblk": false, + "ischr": false, + "isdir": false, + "isfifo": false, + "isgid": false, + "islnk": false, + "isreg": true, + "issock": false, + "isuid": false, + "mimetype": "text/plain", + "mode": "0644", + "mtime": 1753370991.6915624, + "nlink": 1, + "path": "/opt/vdi-setup/.vdi-lock.yaml", + "pw_name": "root", + "readable": true, + "rgrp": true, + "roth": true, + "rusr": true, + "size": 940, + "uid": 0, + "version": "3443573269", + "wgrp": false, + "woth": false, + "writeable": true, + "wusr": true, + "xgrp": false, + "xoth": false, + "xusr": false + } +} + +TASK [lock_manager : Debug lock file existence] ******************************** +ok: [localhost] => {} + +MSG: + +Lock file exists: True + +TASK [lock_manager : Load existing lock file] ********************************** +ok: [localhost] => { + "ansible_facts": { + "vdi_setup_status": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:29:50Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:29:50Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "ansible_included_var_files": [ + "/opt/vdi-setup/.vdi-lock.yaml" + ], + "changed": false +} + +TASK [lock_manager : Debug lock file load result] ****************************** +ok: [localhost] => {} + +MSG: + +Lock file loaded successfully: True + +TASK [lock_manager : Calculate current deployment hash] ************************ +ok: [localhost] => { + "ansible_facts": { + "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab" + }, + "changed": false +} + +TASK [lock_manager : Debug deployment hash calculation] ************************ +ok: [localhost] => {} + +MSG: + +Current deployment hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab +Deployment components: + - deployment_name: vdi-test-scott + - project_id: hpc-discovery-external + - vdi_tool: guacamole + - vnc_flavor: tigervnc + - user_provision: local_users + - vdi_webapp_port: 8081 + - vdi_resolution: 1920x1080 + - vdi_users count: 2 + + +TASK [lock_manager : Calculate current user secrets hash] ********************** +ok: [localhost] => { + "ansible_facts": { + "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" + }, + "changed": false +} + +TASK [lock_manager : Debug user secrets hash calculation] ********************** +ok: [localhost] => {} + +MSG: + +Current user secrets hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 +User configuration components: + - vdi_users (blueprint): [{"username": "alice", "port": 5901}, {"username": "bob", "port": 5902, "secret_name": "a-password-for-bob"}] + + +TASK [lock_manager : Set default values for missing lock file] ***************** +skipping: [localhost] => { + "changed": false, + "false_condition": "not lock_file_stat.stat.exists", + "skip_reason": "Conditional result was False" +} + +TASK [lock_manager : Debug lock file status] *********************************** +ok: [localhost] => {} + +MSG: + +Lock file status: + - exists: True + - deployment_hash: none + - user_secrets_hash: none + - force_rerun: False + + +TASK [lock_manager : Check if lock file exists] ******************************** +ok: [localhost] => { + "ansible_facts": { + "lock_file_exists": true + }, + "changed": false +} + +TASK [lock_manager : Check if current role is completed] *********************** +ok: [localhost] => { + "ansible_facts": { + "current_role_completed": false + }, + "changed": false +} + +TASK [lock_manager : Check if deployment hash matches] ************************* +ok: [localhost] => { + "ansible_facts": { + "deployment_hash_matches": false + }, + "changed": false +} + +TASK [lock_manager : Check if force rerun is enabled] ************************** +ok: [localhost] => { + "ansible_facts": { + "force_rerun_enabled": false + }, + "changed": false +} + +TASK [lock_manager : Determine if role should run] ***************************** +ok: [localhost] => { + "ansible_facts": { + "role_should_run": true + }, + "changed": false +} + +TASK [lock_manager : Debug role should run calculation] ************************ +ok: [localhost] => {} + +MSG: + +Role should run calculation for secret_manager: + - not lock_file_exists: False + - not current_role_completed: True + - not deployment_hash_matches: True + - force_rerun_enabled: False + - role_should_run: True + + +TASK [lock_manager : Debug role execution decision] **************************** +ok: [localhost] => {} + +MSG: + +Role execution decision for secret_manager: + - current_role: secret_manager + - ansible_role_name: lock_manager + - role_should_run: True + - lock_file_exists: True + - current_role_completed: False + - deployment_hash_matches: False + - force_rerun_enabled: False + + +TASK [lock_manager : Check if user configuration has changed] ****************** +ok: [localhost] => { + "ansible_facts": { + "user_config_changed": true + }, + "changed": false +} + +TASK [lock_manager : Set user secrets changed flag (will be updated by secret_manager role)] *** +ok: [localhost] => { + "ansible_facts": { + "user_secrets_changed": true + }, + "changed": false +} + +TASK [lock_manager : Debug user secrets change detection] ********************** +ok: [localhost] => {} + +MSG: + +User configuration change detection: + - user_config_changed: True + - user_secrets_changed: True + - stored_hash: none + - current_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 + + +TASK [lock_manager : Set lock check result] ************************************ +ok: [localhost] => { + "ansible_facts": { + "lock_check_result": { + "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "existing_deployment_hash": "none", + "existing_user_secrets_hash": "none", + "force_rerun": false, + "role_should_run": true, + "user_secrets_changed": true + } + }, + "changed": false +} + +TASK [lock_manager : Debug final lock check result] **************************** +ok: [localhost] => {} + +MSG: + +Final lock check result: + - role_should_run: True + - deployment_hash_changed: True + - user_secrets_changed: True + - force_rerun: False + + +TASK [secret_manager : Set role_should_run from lock check result] ************* +ok: [localhost] => { + "ansible_facts": { + "role_should_run": true + }, + "changed": false +} + +TASK [secret_manager : Skip secret_manager tasks if role should not run] ******* +skipping: [localhost] => { + "false_condition": "not role_should_run" +} + +TASK [secret_manager : Generate random database password] ********************** +ok: [localhost] => { + "ansible_facts": { + "database_password": "H95L0yeLji7pzjDCOuemJDzEwoASP7Nq" + }, + "changed": false +} + +TASK [secret_manager : Generate random webapp admin password] ****************** +ok: [localhost] => { + "ansible_facts": { + "webapp_admin_password": "n7d53VAcYV4xT86sDI8mNKO7BHsIIfa0" + }, + "changed": false +} + +TASK [secret_manager : Compute Webapp admin password hash to save to Webapp server] *** +ok: [localhost] => { + "ansible_facts": { + "webapp_admin_hash": "E4440760F8847690907CADBB2B87C3E21015556DB4DAEA4A0D9C5D5F85DD6EA9" + }, + "changed": false +} + +TASK [secret_manager : Get access token for Secret Manager API] **************** +ok: [localhost] => { + "changed": false, + "connection": "Close", + "content": "{\"access_token\":\"ya29.c.c0ASRK0Gb5OH5v-xtWlMorXSZswjZqBApOSzy3U4RHvOuuIn3lLmwmFBxGPcpUUw72yV0h0ow9Xuo-4544iIzBnhfADz7iDQ09v99UJ3y8RTi6yqwoXjOBcbI1Xqkj1gPMgVsUBGPfooSczQEcQPj27MBL6rLrDadQnacLMKVzogosGtpxH3Kf-a5lztK5AmcyYrgGxfyFqtPbF34J9J-88UghA5vbVdYScappzlkaxWbwoAmK2xW0Z40kE2uz91P2wwgcG1JUAcR6QgOc2_NarPeRkZ1_kOl8COE6l05gNix4hcH5j8PBTsaoUHre-vE-j6gAJRIe2uMUlVa7Mm3CkuJqrfxR3hNXUpsu6mdsTJEDvJz02U4uIMwYrupU58mzflgo1q0al4hW0UEVWwE411P2y-gSmWQY71t8ynep0kpl-alZF3zxk-mhFkfurxl06mj9I9pq7cYb9-Zu3B-S7qv5pmg6ISQQVk33QgXO15n913orUWhFf2qlVtOqMhzgsgF12iB6rwXtts4y1ceodU6JaqkWf3wsF0uQarJwbb7eYg5jBzVjcxwcuy2Y7bhOqfIa0X88nW7_WgqUXXe7FBIQYfJoJ0U1aZjS3Frwicnjr4gMBoW9OQuQcX-ygjUXBmWMXxWpmW5tOnB1xcQxey-jbeypuys1ou0JFIql8Q4iIJyQvWBuchJXaWsyfooRJfr_mXSpf6f2fm0dW1kV3abBx_jIisS7UnUq8hnZyieo515yZ1_ktq-1gicxlum3SltkiSbmYxnJ12vaM3W-t74bi1JRZV2jJess0vFssRrapFzBoQzfUUbuR21hh1ImdmQ6_YlhUh9IJcX-qkk_X24nXuQfdSUptj0WXx14cWs6cfQ_b5-qfnFyymtlcMuS3i7mg3z1vghaf2-24J5IUUYxxOaqJ7_J48pVzaXpJnypnymuU_lp-w88jk_I0zYtUiSuFq5SrsY9_aIQqtOY40pyZdmYO2zg5w4fBpzShjhfpXMI8BicoUz\",\"expires_in\":3599,\"token_type\":\"Bearer\"}", + "content_length": "1083", + "content_type": "application/json", + "cookies": {}, + "cookies_string": "", + "date": "Thu, 24 Jul 2025 15:29:58 GMT", + "elapsed": 0, + "json": { + "access_token": "ya29.c.c0ASRK0Gb5OH5v-xtWlMorXSZswjZqBApOSzy3U4RHvOuuIn3lLmwmFBxGPcpUUw72yV0h0ow9Xuo-4544iIzBnhfADz7iDQ09v99UJ3y8RTi6yqwoXjOBcbI1Xqkj1gPMgVsUBGPfooSczQEcQPj27MBL6rLrDadQnacLMKVzogosGtpxH3Kf-a5lztK5AmcyYrgGxfyFqtPbF34J9J-88UghA5vbVdYScappzlkaxWbwoAmK2xW0Z40kE2uz91P2wwgcG1JUAcR6QgOc2_NarPeRkZ1_kOl8COE6l05gNix4hcH5j8PBTsaoUHre-vE-j6gAJRIe2uMUlVa7Mm3CkuJqrfxR3hNXUpsu6mdsTJEDvJz02U4uIMwYrupU58mzflgo1q0al4hW0UEVWwE411P2y-gSmWQY71t8ynep0kpl-alZF3zxk-mhFkfurxl06mj9I9pq7cYb9-Zu3B-S7qv5pmg6ISQQVk33QgXO15n913orUWhFf2qlVtOqMhzgsgF12iB6rwXtts4y1ceodU6JaqkWf3wsF0uQarJwbb7eYg5jBzVjcxwcuy2Y7bhOqfIa0X88nW7_WgqUXXe7FBIQYfJoJ0U1aZjS3Frwicnjr4gMBoW9OQuQcX-ygjUXBmWMXxWpmW5tOnB1xcQxey-jbeypuys1ou0JFIql8Q4iIJyQvWBuchJXaWsyfooRJfr_mXSpf6f2fm0dW1kV3abBx_jIisS7UnUq8hnZyieo515yZ1_ktq-1gicxlum3SltkiSbmYxnJ12vaM3W-t74bi1JRZV2jJess0vFssRrapFzBoQzfUUbuR21hh1ImdmQ6_YlhUh9IJcX-qkk_X24nXuQfdSUptj0WXx14cWs6cfQ_b5-qfnFyymtlcMuS3i7mg3z1vghaf2-24J5IUUYxxOaqJ7_J48pVzaXpJnypnymuU_lp-w88jk_I0zYtUiSuFq5SrsY9_aIQqtOY40pyZdmYO2zg5w4fBpzShjhfpXMI8BicoUz", + "expires_in": 3599, + "token_type": "Bearer" + }, + "metadata_flavor": "Google", + "redirected": false, + "server": "Metadata Server for VM", + "status": 200, + "url": "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token", + "x_frame_options": "SAMEORIGIN", + "x_xss_protection": "0" +} + +MSG: + +OK (1083 bytes) + +TASK [secret_manager : Create a GCP secret for webapp server login] ************ +ok: [localhost] => { + "accept_ranges": "none", + "changed": false, + "connection": "close", + "content_type": "application/json; charset=UTF-8", + "date": "Thu, 24 Jul 2025 15:29:58 GMT", + "elapsed": 0, + "json": { + "error": { + "code": 409, + "message": "Secret [projects/748006908949/secrets/webapp-server-password-vdi-test-scott] already exists.", + "status": "ALREADY_EXISTS" + } + }, + "redirected": false, + "server": "ESF", + "status": 409, + "transfer_encoding": "chunked", + "url": "https://secretmanager.googleapis.com/v1/projects/hpc-discovery-external/secrets?secretId=webapp-server-password-vdi-test-scott", + "vary": "Origin,Accept-Encoding", + "x_content_type_options": "nosniff", + "x_frame_options": "SAMEORIGIN", + "x_xss_protection": "0" +} + +MSG: + +HTTP Error 409: Conflict + +TASK [secret_manager : Add webapp password version to secret] ****************** +ok: [localhost] => { + "accept_ranges": "none", + "changed": false, + "connection": "close", + "content_type": "application/json; charset=UTF-8", + "cookies": {}, + "cookies_string": "", + "date": "Thu, 24 Jul 2025 15:29:59 GMT", + "elapsed": 0, + "json": { + "createTime": "2025-07-24T15:29:59.522597Z", + "etag": "\"163aae7f960525\"", + "name": "projects/748006908949/secrets/webapp-server-password-vdi-test-scott/versions/82", + "replicationStatus": { + "automatic": {} + }, + "state": "ENABLED" + }, + "redirected": false, + "server": "ESF", + "status": 200, + "transfer_encoding": "chunked", + "url": "https://secretmanager.googleapis.com/v1/projects/hpc-discovery-external/secrets/webapp-server-password-vdi-test-scott:addVersion", + "vary": "X-Origin, Referer, Origin,Accept-Encoding", + "x_content_type_options": "nosniff", + "x_frame_options": "SAMEORIGIN", + "x_xss_protection": "0" +} + +MSG: + +OK (unknown bytes) + +TASK [secret_manager : Create a GCP secret for database password] ************** +ok: [localhost] => { + "accept_ranges": "none", + "changed": false, + "connection": "close", + "content_type": "application/json; charset=UTF-8", + "date": "Thu, 24 Jul 2025 15:29:59 GMT", + "elapsed": 0, + "json": { + "error": { + "code": 409, + "message": "Secret [projects/748006908949/secrets/db-password-vdi-test-scott] already exists.", + "status": "ALREADY_EXISTS" + } + }, + "redirected": false, + "server": "ESF", + "status": 409, + "transfer_encoding": "chunked", + "url": "https://secretmanager.googleapis.com/v1/projects/hpc-discovery-external/secrets?secretId=db-password-vdi-test-scott", + "vary": "Origin,Accept-Encoding", + "x_content_type_options": "nosniff", + "x_frame_options": "SAMEORIGIN", + "x_xss_protection": "0" +} + +MSG: + +HTTP Error 409: Conflict + +TASK [secret_manager : Add database password version to secret] **************** +ok: [localhost] => { + "accept_ranges": "none", + "changed": false, + "connection": "close", + "content_type": "application/json; charset=UTF-8", + "cookies": {}, + "cookies_string": "", + "date": "Thu, 24 Jul 2025 15:30:00 GMT", + "elapsed": 0, + "json": { + "createTime": "2025-07-24T15:30:00.398932Z", + "etag": "\"163aae7fa36454\"", + "name": "projects/748006908949/secrets/db-password-vdi-test-scott/versions/82", + "replicationStatus": { + "automatic": {} + }, + "state": "ENABLED" + }, + "redirected": false, + "server": "ESF", + "status": 200, + "transfer_encoding": "chunked", + "url": "https://secretmanager.googleapis.com/v1/projects/hpc-discovery-external/secrets/db-password-vdi-test-scott:addVersion", + "vary": "X-Origin, Referer, Origin,Accept-Encoding", + "x_content_type_options": "nosniff", + "x_frame_options": "SAMEORIGIN", + "x_xss_protection": "0" +} + +MSG: + +OK (unknown bytes) + +TASK [secret_manager : Generate per-user secrets and enrich list] ************** +included: /tmp/vdi/roles/secret_manager/tasks/user_secret_tasks.yaml for localhost => (item={'username': 'alice', 'port': 5901}) +included: /tmp/vdi/roles/secret_manager/tasks/user_secret_tasks.yaml for localhost => (item={'username': 'bob', 'port': 5902, 'secret_name': 'a-password-for-bob'}) + +TASK [secret_manager : Compute default secret_name if none provided] *********** +ok: [localhost] => { + "ansible_facts": { + "secret_name": "vdi-user-password-alice-vdi-test-scott" + }, + "changed": false +} + +TASK [secret_manager : Determine password source for user alice] *************** +ok: [localhost] => { + "ansible_facts": { + "password_source": "generate" + }, + "changed": false +} + +TASK [secret_manager : Clean password source value] **************************** +ok: [localhost] => { + "ansible_facts": { + "password_source": "generate" + }, + "changed": false +} + +TASK [secret_manager : Debug password source for alice] ************************ +ok: [localhost] => {} + +MSG: + +Password source for alice: + - password_source: "generate" + - secret_name: not set + - password: not set + + +TASK [secret_manager : Fetch existing Secret Manager password for user alice] *** +skipping: [localhost] => { + "changed": false, + "false_condition": "password_source == \"secret\"", + "skip_reason": "Conditional result was False" +} + +TASK [secret_manager : Debug Secret Manager fetch result for alice] ************ +skipping: [localhost] => { + "false_condition": "debug | default(false) and password_source == \"secret\"" +} + +TASK [secret_manager : Set VDI user password from Secret Manager] ************** +skipping: [localhost] => { + "changed": false, + "false_condition": "password_source == \"secret\" and sm_result.status == 200", + "skip_reason": "Conditional result was False" +} + +TASK [secret_manager : Check if password has actually changed for alice] ******* +skipping: [localhost] => { + "changed": false, + "false_condition": "password_source == \"secret\" and sm_result.status == 200", + "skip_reason": "Conditional result was False" +} + +TASK [secret_manager : Debug password change detection for alice] ************** +skipping: [localhost] => { + "false_condition": "debug | default(false) and password_source == \"secret\"" +} + +TASK [secret_manager : Generate random password for user alice] **************** +ok: [localhost] => { + "ansible_facts": { + "vdiuser_password": "xc61q4jU3mUCo84M" + }, + "changed": false +} + +TASK [secret_manager : Set user password from provided value] ****************** +skipping: [localhost] => { + "changed": false, + "false_condition": "password_source == \"password\"", + "skip_reason": "Conditional result was False" +} + +TASK [secret_manager : Create secret for user alice (provided password)] ******* +skipping: [localhost] => { + "changed": false, + "false_condition": "password_source == \"password\"", + "skip_reason": "Conditional result was False" +} + +TASK [secret_manager : Store provided password in Secret Manager for user alice] *** +skipping: [localhost] => { + "changed": false, + "false_condition": "password_source == \"password\"", + "skip_reason": "Conditional result was False" +} + +TASK [secret_manager : Create secret for user alice] *************************** +ok: [localhost] => { + "accept_ranges": "none", + "changed": false, + "connection": "close", + "content_type": "application/json; charset=UTF-8", + "date": "Thu, 24 Jul 2025 15:30:01 GMT", + "elapsed": 0, + "json": { + "error": { + "code": 409, + "message": "Secret [projects/748006908949/secrets/vdi-user-password-alice-vdi-test-scott] already exists.", + "status": "ALREADY_EXISTS" + } + }, + "redirected": false, + "server": "ESF", + "status": 409, + "transfer_encoding": "chunked", + "url": "https://secretmanager.googleapis.com/v1/projects/hpc-discovery-external/secrets?secretId=vdi-user-password-alice-vdi-test-scott", + "vary": "Origin,Accept-Encoding", + "x_content_type_options": "nosniff", + "x_frame_options": "SAMEORIGIN", + "x_xss_protection": "0" +} + +MSG: + +HTTP Error 409: Conflict + +TASK [secret_manager : Store generated password in Secret Manager for user alice] *** +ok: [localhost] => { + "accept_ranges": "none", + "changed": false, + "connection": "close", + "content_type": "application/json; charset=UTF-8", + "cookies": {}, + "cookies_string": "", + "date": "Thu, 24 Jul 2025 15:30:01 GMT", + "elapsed": 0, + "json": { + "createTime": "2025-07-24T15:30:01.698173Z", + "etag": "\"163aae7fb7377d\"", + "name": "projects/748006908949/secrets/vdi-user-password-alice-vdi-test-scott/versions/63", + "replicationStatus": { + "automatic": {} + }, + "state": "ENABLED" + }, + "redirected": false, + "server": "ESF", + "status": 200, + "transfer_encoding": "chunked", + "url": "https://secretmanager.googleapis.com/v1/projects/hpc-discovery-external/secrets/vdi-user-password-alice-vdi-test-scott:addVersion", + "vary": "X-Origin, Referer, Origin,Accept-Encoding", + "x_content_type_options": "nosniff", + "x_frame_options": "SAMEORIGIN", + "x_xss_protection": "0" +} + +MSG: + +OK (unknown bytes) + +TASK [secret_manager : Set VNC configuration for user] ************************* +ok: [localhost] => { + "ansible_facts": { + "vnc_config": { + "display_number": 1, + "vncserver_password": "4JB1YHUQ" + } + }, + "changed": false +} + +TASK [secret_manager : Build enriched user list] ******************************* +ok: [localhost] => { + "ansible_facts": { + "vdi_users_updated": [ + { + "display_number": 1, + "password": "xc61q4jU3mUCo84M", + "port": 5901, + "username": "alice", + "vncserver_password": "4JB1YHUQ" + } + ] + }, + "changed": false +} + +TASK [secret_manager : Compute default secret_name if none provided] *********** +skipping: [localhost] => { + "changed": false, + "false_condition": "item.secret_name is not defined", + "skip_reason": "Conditional result was False" +} + +TASK [secret_manager : Determine password source for user bob] ***************** +ok: [localhost] => { + "ansible_facts": { + "password_source": "secret" + }, + "changed": false +} + +TASK [secret_manager : Clean password source value] **************************** +ok: [localhost] => { + "ansible_facts": { + "password_source": "secret" + }, + "changed": false +} + +TASK [secret_manager : Debug password source for bob] ************************** +ok: [localhost] => {} + +MSG: + +Password source for bob: + - password_source: "secret" + - secret_name: a-password-for-bob + - password: not set + + +TASK [secret_manager : Fetch existing Secret Manager password for user bob] **** +ok: [localhost] => { + "accept_ranges": "none", + "changed": false, + "connection": "close", + "content": "{\n \"name\": \"projects/748006908949/secrets/a-password-for-bob/versions/4\",\n \"payload\": {\n \"data\": \"QjBicDRzc3cwcmQxMjM=\",\n \"dataCrc32c\": \"3219829350\"\n }\n}\n", + "content_type": "application/json; charset=UTF-8", + "cookies": {}, + "cookies_string": "", + "date": "Thu, 24 Jul 2025 15:30:02 GMT", + "elapsed": 0, + "failed_when_result": false, + "json": { + "name": "projects/748006908949/secrets/a-password-for-bob/versions/4", + "payload": { + "data": "QjBicDRzc3cwcmQxMjM=", + "dataCrc32c": "3219829350" + } + }, + "redirected": false, + "server": "ESF", + "status": 200, + "transfer_encoding": "chunked", + "url": "https://secretmanager.googleapis.com/v1/projects/hpc-discovery-external/secrets/a-password-for-bob/versions/latest:access", + "vary": "X-Origin, Referer, Origin,Accept-Encoding", + "x_content_type_options": "nosniff", + "x_frame_options": "SAMEORIGIN", + "x_xss_protection": "0" +} + +MSG: + +OK (unknown bytes) + +TASK [secret_manager : Debug Secret Manager fetch result for bob] ************** +ok: [localhost] => {} + +MSG: + +Secret Manager fetch for bob: + - status: 200 + - success: True + - user_config_changed: True + - password_source: secret + + +TASK [secret_manager : Set VDI user password from Secret Manager] ************** +ok: [localhost] => { + "ansible_facts": { + "vdiuser_password": "B0bp4ssw0rd123" + }, + "changed": false +} + +TASK [secret_manager : Check if password has actually changed for bob] ********* +ok: [localhost] => { + "ansible_facts": { + "password_actually_changed": true + }, + "changed": false +} + +TASK [secret_manager : Debug password change detection for bob] **************** +ok: [localhost] => {} + +MSG: + +Password change detection for bob: + - user_config_changed: True + - password_source: secret + - sm_result_status: 200 + - password_actually_changed: True + + +TASK [secret_manager : Generate random password for user bob] ****************** +skipping: [localhost] => { + "changed": false, + "false_condition": "password_source == \"generate\"", + "skip_reason": "Conditional result was False" +} + +TASK [secret_manager : Set user password from provided value] ****************** +skipping: [localhost] => { + "changed": false, + "false_condition": "password_source == \"password\"", + "skip_reason": "Conditional result was False" +} + +TASK [secret_manager : Create secret for user bob (provided password)] ********* +skipping: [localhost] => { + "changed": false, + "false_condition": "password_source == \"password\"", + "skip_reason": "Conditional result was False" +} + +TASK [secret_manager : Store provided password in Secret Manager for user bob] *** +skipping: [localhost] => { + "changed": false, + "false_condition": "password_source == \"password\"", + "skip_reason": "Conditional result was False" +} + +TASK [secret_manager : Create secret for user bob] ***************************** +skipping: [localhost] => { + "changed": false, + "false_condition": "password_source == \"generate\"", + "skip_reason": "Conditional result was False" +} + +TASK [secret_manager : Store generated password in Secret Manager for user bob] *** +skipping: [localhost] => { + "changed": false, + "false_condition": "password_source == \"generate\"", + "skip_reason": "Conditional result was False" +} + +TASK [secret_manager : Set VNC configuration for user] ************************* +ok: [localhost] => { + "ansible_facts": { + "vnc_config": { + "display_number": 2, + "vncserver_password": "7DmxvFp2" + } + }, + "changed": false +} + +TASK [secret_manager : Build enriched user list] ******************************* +ok: [localhost] => { + "ansible_facts": { + "vdi_users_updated": [ + { + "display_number": 1, + "password": "xc61q4jU3mUCo84M", + "port": 5901, + "username": "alice", + "vncserver_password": "4JB1YHUQ" + }, + { + "display_number": 2, + "password": "B0bp4ssw0rd123", + "port": 5902, + "secret_name": "a-password-for-bob", + "username": "bob", + "vncserver_password": "7DmxvFp2" + } + ] + }, + "changed": false +} + +TASK [secret_manager : Set enriched vdi_users_updated variable] **************** +ok: [localhost] => { + "ansible_facts": { + "vdi_users_updated": [ + { + "display_number": 1, + "password": "xc61q4jU3mUCo84M", + "port": 5901, + "username": "alice", + "vncserver_password": "4JB1YHUQ" + }, + { + "display_number": 2, + "password": "B0bp4ssw0rd123", + "port": 5902, + "secret_name": "a-password-for-bob", + "username": "bob", + "vncserver_password": "7DmxvFp2" + } + ] + }, + "changed": false +} + +TASK [secret_manager : Set current role for lock manager completion] *********** +ok: [localhost] => { + "ansible_facts": { + "current_role": "secret_manager", + "role_completed": true + }, + "changed": false +} + +TASK [lock_manager : Debug create lock operation] ****************************** +ok: [localhost] => {} + +MSG: + +Creating/updating lock file for role: secret_manager + +TASK [lock_manager : Check if lock file exists] ******************************** +ok: [localhost] => { + "changed": false, + "stat": { + "atime": 1753370992.073591, + "attr_flags": "", + "attributes": [], + "block_size": 4096, + "blocks": 8, + "charset": "us-ascii", + "checksum": "9dd77a3585697aef08aed09e819bb4819f75aeb3", + "ctime": 1753370991.845574, + "dev": 2050, + "device_type": 0, + "executable": false, + "exists": true, + "gid": 0, + "gr_name": "root", + "inode": 185115083, + "isblk": false, + "ischr": false, + "isdir": false, + "isfifo": false, + "isgid": false, + "islnk": false, + "isreg": true, + "issock": false, + "isuid": false, + "mimetype": "text/plain", + "mode": "0644", + "mtime": 1753370991.6915624, + "nlink": 1, + "path": "/opt/vdi-setup/.vdi-lock.yaml", + "pw_name": "root", + "readable": true, + "rgrp": true, + "roth": true, + "rusr": true, + "size": 940, + "uid": 0, + "version": "3443573269", + "wgrp": false, + "woth": false, + "writeable": true, + "wusr": true, + "xgrp": false, + "xoth": false, + "xusr": false + } +} + +TASK [lock_manager : Debug lock file existence] ******************************** +ok: [localhost] => {} + +MSG: + +Lock file exists: True + +TASK [lock_manager : Create initial lock file structure] *********************** +skipping: [localhost] => { + "changed": false, + "false_condition": "not lock_file_stat.stat.exists", + "skip_reason": "Conditional result was False" +} + +TASK [lock_manager : Debug lock file creation] ********************************* +skipping: [localhost] => { + "false_condition": "debug | default(false) and not lock_file_stat.stat.exists" +} + +TASK [lock_manager : Load existing lock file data] ***************************** +ok: [localhost] => { + "ansible_facts": { + "vdi_setup_status": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:29:50Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:29:50Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "ansible_included_var_files": [ + "/opt/vdi-setup/.vdi-lock.yaml" + ], + "changed": false +} + +TASK [lock_manager : Get current lock data] ************************************ +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:29:50Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:29:50Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Update deployment hash] *********************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:29:50Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:29:50Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Get current timestamp] ************************************ +changed: [localhost] => { + "changed": true, + "cmd": [ + "date", + "-u", + "+%Y-%m-%dT%H:%M:%SZ" + ], + "delta": "0:00:00.002549", + "end": "2025-07-24 15:30:03.439894", + "rc": 0, + "start": "2025-07-24 15:30:03.437345" +} + +STDOUT: + +2025-07-24T15:30:03Z + +TASK [lock_manager : Update last updated timestamp] **************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:03Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:29:50Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Get current user secrets status] ************************** +ok: [localhost] => { + "ansible_facts": { + "current_user_secrets": { + "last_secret_check": "2025-07-24T15:29:50Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Create user secrets update] ******************************* +ok: [localhost] => { + "ansible_facts": { + "user_secrets_update": { + "last_secret_check": "2025-07-24T15:30:03Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" + } + }, + "changed": false +} + +TASK [lock_manager : Update user secrets hash] ********************************* +ok: [localhost] => { + "ansible_facts": { + "updated_user_secrets": { + "last_secret_check": "2025-07-24T15:30:03Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Apply user secrets update] ******************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:03Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:03Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Create role status entry] ********************************* +ok: [localhost] => { + "ansible_facts": { + "role_status_entry": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + } + }, + "changed": false +} + +TASK [lock_manager : Debug role status entry] ********************************** +ok: [localhost] => {} + +MSG: + +Role status entry for secret_manager: + - completed: True + - completed_at: 2025-07-24T15:30:03Z + + +TASK [lock_manager : Debug current lock data before role update] *************** +ok: [localhost] => {} + +MSG: + +Current lock data before updating secret_manager: + - current_lock_data: completed_roles: + base_os: + completed: true + completed_at: '2025-07-24T15:29:50Z' + lock_manager: + completed: true + completed_at: '2025-07-24T15:27:44Z' + secret_manager: + completed: false + user_provision: + completed: false + vdi_tool: + completed: false + vnc: + completed: false +created_at: '2025-07-24T15:27:41Z' +deployment_hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab +deployment_name: vdi-test-scott +force_rerun: false +last_updated: '2025-07-24T15:30:03Z' +lock_version: '1.0' +setup_status: configuring +user_management: + current_users: + - alice + - bob + previous_users: + - alice + - bob + removed_users: [] +user_secrets_status: + last_secret_check: '2025-07-24T15:30:03Z' + user_secrets_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 + users_updated: + - alice + - bob + + - current_completed_roles: base_os: + completed: true + completed_at: '2025-07-24T15:29:50Z' +lock_manager: + completed: true + completed_at: '2025-07-24T15:27:44Z' +secret_manager: + completed: false +user_provision: + completed: false +vdi_tool: + completed: false +vnc: + completed: false + + - role_completed: True + - current_role: secret_manager + + +TASK [lock_manager : Get current completed roles] ****************************** +ok: [localhost] => { + "ansible_facts": { + "current_completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": false + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + } + }, + "changed": false +} + +TASK [lock_manager : Update completed roles] *********************************** +ok: [localhost] => { + "ansible_facts": { + "updated_completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + } + }, + "changed": false +} + +TASK [lock_manager : Update lock data with completed roles] ******************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:03Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:03Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Debug updated lock data after role update] **************** +ok: [localhost] => {} + +MSG: + +Updated lock data after updating secret_manager: + - current_role: secret_manager + - role_completed: True + - completed_roles: {'base_os': {'completed': True, 'completed_at': '2025-07-24T15:29:50Z'}, 'lock_manager': {'completed': True, 'completed_at': '2025-07-24T15:27:44Z'}, 'secret_manager': {'completed': True, 'completed_at': '2025-07-24T15:30:03Z'}, 'user_provision': {'completed': False}, 'vdi_tool': {'completed': False}, 'vnc': {'completed': False}} + + +TASK [lock_manager : Get current users updated list] *************************** +ok: [localhost] => { + "ansible_facts": { + "current_users_updated": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Get new usernames list] *********************************** +ok: [localhost] => { + "ansible_facts": { + "new_usernames": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Combine users updated lists] ****************************** +ok: [localhost] => { + "ansible_facts": { + "combined_users_updated": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Debug user deduplication] ********************************* +ok: [localhost] => {} + +MSG: + +User deduplication for secret_manager: + - current_users_updated: ['alice', 'bob'] + - new_usernames: ['alice', 'bob'] + - combined_users_updated: ['alice', 'bob'] + + +TASK [lock_manager : Get current user secrets for list update] ***************** +ok: [localhost] => { + "ansible_facts": { + "current_user_secrets_for_list": { + "last_secret_check": "2025-07-24T15:30:03Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Update users updated list] ******************************** +ok: [localhost] => { + "ansible_facts": { + "updated_user_secrets_with_list": { + "last_secret_check": "2025-07-24T15:30:03Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Apply users updated list] ********************************* +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:03Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:03Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Get current user management data] ************************* +ok: [localhost] => { + "ansible_facts": { + "current_user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + } + }, + "changed": false +} + +TASK [lock_manager : Get current usernames list] ******************************* +ok: [localhost] => { + "ansible_facts": { + "current_usernames": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Update user management data] ****************************** +ok: [localhost] => { + "ansible_facts": { + "updated_user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + } + }, + "changed": false +} + +TASK [lock_manager : Apply user management update] ***************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:03Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:03Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Debug user management] ************************************ +ok: [localhost] => {} + +MSG: + +User management for secret_manager: + - current_users: ['alice', 'bob'] + - previous_users: ['alice', 'bob'] + - removed_users: [] + + +TASK [lock_manager : Check if all roles are completed] ************************* +ok: [localhost] => { + "ansible_facts": { + "all_roles_completed": false + }, + "changed": false +} + +TASK [lock_manager : Update setup status based on completion] ****************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:03Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:03Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Debug setup status update] ******************************** +ok: [localhost] => {} + +MSG: + +Setup status update: + - all_roles_completed: False + - setup_status: configuring + - completed_roles_count: 3 + - total_roles_count: 6 + + +TASK [lock_manager : Write updated lock file] ********************************** +changed: [localhost] => { + "changed": true, + "checksum": "9205f17d8ce64bf55a046c433f6f24c4efa12eae", + "dest": "/opt/vdi-setup/.vdi-lock.yaml", + "gid": 0, + "group": "root", + "md5sum": "8a1ac8aa138bd51d4b3fd6f087ff52cb", + "mode": "0644", + "owner": "root", + "size": 982, + "src": "/root/.ansible/tmp/ansible-tmp-1753371004.5096657-25387-190441536237874/source", + "state": "file", + "uid": 0 +} + +TASK [lock_manager : Debug VM metadata update] ********************************* +ok: [localhost] => {} + +MSG: + +VM metadata update info: + - hostname: vdi-test-scott-rocky-0 + - zone: us-central1-a + - lock_file_changed: True + + +TASK [lock_manager : Get lock file content as base64] ************************** +changed: [localhost] => { + "changed": true, + "cmd": "cat /opt/vdi-setup/.vdi-lock.yaml | base64 -w 0", + "delta": "0:00:00.003695", + "end": "2025-07-24 15:30:05.054073", + "rc": 0, + "start": "2025-07-24 15:30:05.050378" +} + +STDOUT: + +dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiBmYWxzZQogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogZmFsc2UKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiBmYWxzZQogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogY29uZmlndXJpbmcKICB1c2VyX21hbmFnZW1lbnQ6CiAgICBjdXJyZW50X3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHByZXZpb3VzX3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHJlbW92ZWRfdXNlcnM6IFtdCiAgdXNlcl9zZWNyZXRzX3N0YXR1czoKICAgIGxhc3Rfc2VjcmV0X2NoZWNrOiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgICB1c2VyX3NlY3JldHNfaGFzaDogNmE2YTAyZDNkMDk5MzNjYTI4MmFlMmIwMjMzNjM0M2M5ZDczNzAxZTQ1YmQ4MTQ3NWZiZDkyZGNmOWQyZjNiNgogICAgdXNlcnNfdXBkYXRlZDoKICAgIC0gYWxpY2UKICAgIC0gYm9iCg== + +TASK [lock_manager : Add VDI lock file content to VM metadata] ***************** +changed: [localhost] => { + "changed": true, + "cmd": [ + "gcloud", + "compute", + "instances", + "add-metadata", + "vdi-test-scott-rocky-0", + "--metadata", + "vdi-lock-content=dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiBmYWxzZQogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogZmFsc2UKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiBmYWxzZQogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogY29uZmlndXJpbmcKICB1c2VyX21hbmFnZW1lbnQ6CiAgICBjdXJyZW50X3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHByZXZpb3VzX3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHJlbW92ZWRfdXNlcnM6IFtdCiAgdXNlcl9zZWNyZXRzX3N0YXR1czoKICAgIGxhc3Rfc2VjcmV0X2NoZWNrOiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgICB1c2VyX3NlY3JldHNfaGFzaDogNmE2YTAyZDNkMDk5MzNjYTI4MmFlMmIwMjMzNjM0M2M5ZDczNzAxZTQ1YmQ4MTQ3NWZiZDkyZGNmOWQyZjNiNgogICAgdXNlcnNfdXBkYXRlZDoKICAgIC0gYWxpY2UKICAgIC0gYm9iCg==", + "--zone=us-central1-a" + ], + "delta": "0:00:04.452715", + "end": "2025-07-24 15:30:09.703135", + "failed_when_result": false, + "rc": 0, + "start": "2025-07-24 15:30:05.250420" +} + +STDERR: + +Updated [https://www.googleapis.com/compute/v1/projects/hpc-discovery-external/zones/us-central1-a/instances/vdi-test-scott-rocky-0]. + +TASK [lock_manager : Debug lock file write] ************************************ +ok: [localhost] => {} + +MSG: + +Lock file written: True + +TASK [lock_manager : Verify lock file integrity] ******************************* +ok: [localhost] => { + "changed": false, + "cmd": [ + "python3", + "-c", + "import yaml; yaml.safe_load(open('/opt/vdi-setup/.vdi-lock.yaml'))" + ], + "delta": "0:00:00.039944", + "end": "2025-07-24 15:30:09.991625", + "rc": 0, + "start": "2025-07-24 15:30:09.951681" +} + +TASK [lock_manager : Debug lock file validation] ******************************* +ok: [localhost] => {} + +MSG: + +Lock file validation: True + +TASK [lock_manager : Check if VDI monitor service exists] ********************** +ok: [localhost] => { + "changed": false, + "stat": { + "atime": 1753370874.1548011, + "attr_flags": "", + "attributes": [], + "block_size": 4096, + "blocks": 8, + "charset": "us-ascii", + "checksum": "f6de22707f8d9248014f679a2f1930451e2bf635", + "ctime": 1753370872.6256874, + "dev": 2050, + "device_type": 0, + "executable": false, + "exists": true, + "gid": 0, + "gr_name": "root", + "inode": 285214140, + "isblk": false, + "ischr": false, + "isdir": false, + "isfifo": false, + "isgid": false, + "islnk": false, + "isreg": true, + "issock": false, + "isuid": false, + "mimetype": "text/plain", + "mode": "0644", + "mtime": 1753370872.4766762, + "nlink": 1, + "path": "/etc/systemd/system/vdi-monitor.service", + "pw_name": "root", + "readable": true, + "rgrp": true, + "roth": true, + "rusr": true, + "size": 283, + "uid": 0, + "version": "1338188335", + "wgrp": false, + "woth": false, + "writeable": true, + "wusr": true, + "xgrp": false, + "xoth": false, + "xusr": false + } +} + +TASK [lock_manager : Debug VDI monitor setup conditions] *********************** +ok: [localhost] => {} + +MSG: + +VDI monitor setup conditions: + - current_role: secret_manager + - vdi_monitor_service_exists: True + - lock_file_creation.changed: False + - will_setup_monitor: False + + +TASK [lock_manager : Setup VDI monitoring service (first time only)] *********** +skipping: [localhost] => { + "changed": false, + "false_condition": "not vdi_monitor_service_stat.stat.exists", + "skip_reason": "Conditional result was False" +} + +TASK [lock_manager : Fail if lock file is invalid] ***************************** +skipping: [localhost] => { + "changed": false, + "false_condition": "lock_file_validation.rc != 0", + "skip_reason": "Conditional result was False" +} + +TASK [user_provision : Set current role for lock manager check] **************** +ok: [localhost] => { + "ansible_facts": { + "current_role": "user_provision" + }, + "changed": false +} + +TASK [lock_manager : Check if lock file exists] ******************************** +ok: [localhost] => { + "changed": false, + "stat": { + "atime": 1753371005.05256, + "attr_flags": "", + "attributes": [], + "block_size": 4096, + "blocks": 8, + "charset": "us-ascii", + "checksum": "9205f17d8ce64bf55a046c433f6f24c4efa12eae", + "ctime": 1753371004.8265433, + "dev": 2050, + "device_type": 0, + "executable": false, + "exists": true, + "gid": 0, + "gr_name": "root", + "inode": 51539232, + "isblk": false, + "ischr": false, + "isdir": false, + "isfifo": false, + "isgid": false, + "islnk": false, + "isreg": true, + "issock": false, + "isuid": false, + "mimetype": "text/plain", + "mode": "0644", + "mtime": 1753371004.6825325, + "nlink": 1, + "path": "/opt/vdi-setup/.vdi-lock.yaml", + "pw_name": "root", + "readable": true, + "rgrp": true, + "roth": true, + "rusr": true, + "size": 982, + "uid": 0, + "version": "2720298527", + "wgrp": false, + "woth": false, + "writeable": true, + "wusr": true, + "xgrp": false, + "xoth": false, + "xusr": false + } +} + +TASK [lock_manager : Debug lock file existence] ******************************** +ok: [localhost] => {} + +MSG: + +Lock file exists: True + +TASK [lock_manager : Load existing lock file] ********************************** +ok: [localhost] => { + "ansible_facts": { + "vdi_setup_status": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:03Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:03Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "ansible_included_var_files": [ + "/opt/vdi-setup/.vdi-lock.yaml" + ], + "changed": false +} + +TASK [lock_manager : Debug lock file load result] ****************************** +ok: [localhost] => {} + +MSG: + +Lock file loaded successfully: True + +TASK [lock_manager : Calculate current deployment hash] ************************ +ok: [localhost] => { + "ansible_facts": { + "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab" + }, + "changed": false +} + +TASK [lock_manager : Debug deployment hash calculation] ************************ +ok: [localhost] => {} + +MSG: + +Current deployment hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab +Deployment components: + - deployment_name: vdi-test-scott + - project_id: hpc-discovery-external + - vdi_tool: guacamole + - vnc_flavor: tigervnc + - user_provision: local_users + - vdi_webapp_port: 8081 + - vdi_resolution: 1920x1080 + - vdi_users count: 2 + + +TASK [lock_manager : Calculate current user secrets hash] ********************** +ok: [localhost] => { + "ansible_facts": { + "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" + }, + "changed": false +} + +TASK [lock_manager : Debug user secrets hash calculation] ********************** +ok: [localhost] => {} + +MSG: + +Current user secrets hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 +User configuration components: + - vdi_users (blueprint): [{"username": "alice", "port": 5901}, {"username": "bob", "port": 5902, "secret_name": "a-password-for-bob"}] + + +TASK [lock_manager : Set default values for missing lock file] ***************** +skipping: [localhost] => { + "changed": false, + "false_condition": "not lock_file_stat.stat.exists", + "skip_reason": "Conditional result was False" +} + +TASK [lock_manager : Debug lock file status] *********************************** +ok: [localhost] => {} + +MSG: + +Lock file status: + - exists: True + - deployment_hash: none + - user_secrets_hash: none + - force_rerun: False + + +TASK [lock_manager : Check if lock file exists] ******************************** +ok: [localhost] => { + "ansible_facts": { + "lock_file_exists": true + }, + "changed": false +} + +TASK [lock_manager : Check if current role is completed] *********************** +ok: [localhost] => { + "ansible_facts": { + "current_role_completed": false + }, + "changed": false +} + +TASK [lock_manager : Check if deployment hash matches] ************************* +ok: [localhost] => { + "ansible_facts": { + "deployment_hash_matches": false + }, + "changed": false +} + +TASK [lock_manager : Check if force rerun is enabled] ************************** +ok: [localhost] => { + "ansible_facts": { + "force_rerun_enabled": false + }, + "changed": false +} + +TASK [lock_manager : Determine if role should run] ***************************** +ok: [localhost] => { + "ansible_facts": { + "role_should_run": true + }, + "changed": false +} + +TASK [lock_manager : Debug role should run calculation] ************************ +ok: [localhost] => {} + +MSG: + +Role should run calculation for user_provision: + - not lock_file_exists: False + - not current_role_completed: True + - not deployment_hash_matches: True + - force_rerun_enabled: False + - role_should_run: True + + +TASK [lock_manager : Debug role execution decision] **************************** +ok: [localhost] => {} + +MSG: + +Role execution decision for user_provision: + - current_role: user_provision + - ansible_role_name: lock_manager + - role_should_run: True + - lock_file_exists: True + - current_role_completed: False + - deployment_hash_matches: False + - force_rerun_enabled: False + + +TASK [lock_manager : Check if user configuration has changed] ****************** +ok: [localhost] => { + "ansible_facts": { + "user_config_changed": true + }, + "changed": false +} + +TASK [lock_manager : Set user secrets changed flag (will be updated by secret_manager role)] *** +ok: [localhost] => { + "ansible_facts": { + "user_secrets_changed": true + }, + "changed": false +} + +TASK [lock_manager : Debug user secrets change detection] ********************** +ok: [localhost] => {} + +MSG: + +User configuration change detection: + - user_config_changed: True + - user_secrets_changed: True + - stored_hash: none + - current_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 + + +TASK [lock_manager : Set lock check result] ************************************ +ok: [localhost] => { + "ansible_facts": { + "lock_check_result": { + "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "existing_deployment_hash": "none", + "existing_user_secrets_hash": "none", + "force_rerun": false, + "role_should_run": true, + "user_secrets_changed": true + } + }, + "changed": false +} + +TASK [lock_manager : Debug final lock check result] **************************** +ok: [localhost] => {} + +MSG: + +Final lock check result: + - role_should_run: True + - deployment_hash_changed: True + - user_secrets_changed: True + - force_rerun: False + + +TASK [user_provision : Set role_should_run from lock check result] ************* +ok: [localhost] => { + "ansible_facts": { + "role_should_run": true + }, + "changed": false +} + +TASK [user_provision : Skip user_provision tasks if role should not run] ******* +skipping: [localhost] => { + "false_condition": "not role_should_run" +} + +TASK [user_provision : Ensure 'wheel' group is present] ************************ +ok: [localhost] => { + "changed": false, + "gid": 10, + "name": "wheel", + "state": "present", + "system": false +} + +TASK [user_provision : Ensure VDI user group exists] *************************** +changed: [localhost] => { + "changed": true, + "gid": 1001, + "name": "vdiusers", + "state": "present", + "system": false +} + +TASK [user_provision : Provision local users] ********************************** +included: /tmp/vdi/roles/user_provision/tasks/local_users.yaml for localhost => (item={'username': 'alice', 'port': 5901, 'password': 'xc61q4jU3mUCo84M', 'vncserver_password': '4JB1YHUQ', 'display_number': 1}) +included: /tmp/vdi/roles/user_provision/tasks/local_users.yaml for localhost => (item={'username': 'bob', 'port': 5902, 'secret_name': 'a-password-for-bob', 'password': 'B0bp4ssw0rd123', 'vncserver_password': '7DmxvFp2', 'display_number': 2}) + +TASK [user_provision : Check if user exists] *********************************** +fatal: [localhost]: FAILED! => { + "changed": false +} + +MSG: + +One or more supplied key could not be found in the database. +...ignoring + +TASK [user_provision : Set user exists status for debug] *********************** +ok: [localhost] => { + "ansible_facts": { + "user_exists_status": false + }, + "changed": false +} + +TASK [user_provision : Set user exists status for debug (success case)] ******** +skipping: [localhost] => { + "changed": false, + "false_condition": "debug | default(false) and user_exists_check is not failed", + "skip_reason": "Conditional result was False" +} + +TASK [user_provision : Debug user provision for alice] ************************* +ok: [localhost] => {} + +MSG: + +Provisioning user alice: + - user_secrets_changed: True + - password_actually_changed: True + - password_update_mode: always + - user_exists: False + + +TASK [user_provision : Create VDI user alice] ********************************** +[DEPRECATION WARNING]: Encryption using the Python crypt module is deprecated. +The Python crypt module is deprecated and will be removed from Python 3.13. +Install the passlib library for continued encryption functionality. This +feature will be removed in version 2.17. Deprecation warnings can be disabled +by setting deprecation_warnings=False in ansible.cfg. +changed: [localhost] => { + "changed": true, + "comment": "", + "create_home": true, + "group": 1002, + "groups": "vdiusers,wheel", + "home": "/home/alice", + "name": "alice", + "password": "NOT_LOGGING_PASSWORD", + "shell": "/bin/bash", + "state": "present", + "system": false, + "uid": 1000 +} + +TASK [user_provision : Ensure .ssh directory for alice] ************************ +changed: [localhost] => { + "changed": true, + "gid": 1001, + "group": "vdiusers", + "mode": "0700", + "owner": "alice", + "path": "/home/alice/.ssh", + "size": 6, + "state": "directory", + "uid": 1000 +} + +TASK [user_provision : Generate SSH key pair for alice] ************************ +changed: [localhost] => { + "changed": true, + "comment": "", + "filename": "/home/alice/.ssh/id_rsa", + "fingerprint": "SHA256:4q3JaXL5hMpzved5z0zkLeFXpxKwhFioOkCgEo9i81A", + "public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCxte3GQvt8uFeofNaIUfEMGldCrg9DN8/rUTHzq/dicUI3WuaQKWMC+22TAd7xMernwPlsCUfl3+o5RI1g8vwOGyXmF8dlXKbcokBL2976Ek+7jsibyqzcGiBRKPpN2Rc37L6VMaXupQ+kor/d0zSJXfLVewaWoQcOCktTVFwek7mpVHDbwBAFHN9ruR4M7ItDaJg6fOpD3/cgaegz+L0IIATbXXYGxYTQXnRMEvydcsDFWY8bXKON+YBtZhglc8Y7DJadM2J3S7ZxGwCU2dA4dTRYDTHIeLno0CbWul++qnyRW3OHj0DtdYqtrwi/CnstwYmJ5E9vPctdcGJokw1CYKomgrlpernuRytxrlt2SA6UHyX9VfSiBSuRDkMsyLF3Y/m/nYiDwDjp68XStRt6TmkZNb6zzJnKcRjKmaNiRi5Q1fBULcW4+UWC17SoMX4a74+6WMAtWwDVZvC+UwiCHEJRCKmAMFEE1EDMSsBLX223qXEBIDaXGoyErbezNr3ZUUPR5ZABIfhQW/LwGMW1+WE+k/pFoKeMicHc0EpdNfcZoHEqHKPoKsZl0U8FWJVjLWQO8jVQlUHLhapzA/6vU/JHnpRZ1PrUH6v0PNPjgDhoaaOo7HfdlVKJFNuPV1jC5re1lmBHnZLzyLr8dojaAjElnTr8RRKRHE2X8KTg4w==", + "size": 4096, + "type": "rsa" +} + +TASK [user_provision : Add public key to authorized_keys for alice] ************ +changed: [localhost] => { + "changed": true, + "comment": null, + "exclusive": false, + "follow": false, + "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCxte3GQvt8uFeofNaIUfEMGldCrg9DN8/rUTHzq/dicUI3WuaQKWMC+22TAd7xMernwPlsCUfl3+o5RI1g8vwOGyXmF8dlXKbcokBL2976Ek+7jsibyqzcGiBRKPpN2Rc37L6VMaXupQ+kor/d0zSJXfLVewaWoQcOCktTVFwek7mpVHDbwBAFHN9ruR4M7ItDaJg6fOpD3/cgaegz+L0IIATbXXYGxYTQXnRMEvydcsDFWY8bXKON+YBtZhglc8Y7DJadM2J3S7ZxGwCU2dA4dTRYDTHIeLno0CbWul++qnyRW3OHj0DtdYqtrwi/CnstwYmJ5E9vPctdcGJokw1CYKomgrlpernuRytxrlt2SA6UHyX9VfSiBSuRDkMsyLF3Y/m/nYiDwDjp68XStRt6TmkZNb6zzJnKcRjKmaNiRi5Q1fBULcW4+UWC17SoMX4a74+6WMAtWwDVZvC+UwiCHEJRCKmAMFEE1EDMSsBLX223qXEBIDaXGoyErbezNr3ZUUPR5ZABIfhQW/LwGMW1+WE+k/pFoKeMicHc0EpdNfcZoHEqHKPoKsZl0U8FWJVjLWQO8jVQlUHLhapzA/6vU/JHnpRZ1PrUH6v0PNPjgDhoaaOo7HfdlVKJFNuPV1jC5re1lmBHnZLzyLr8dojaAjElnTr8RRKRHE2X8KTg4w==", + "key_options": null, + "keyfile": "/home/alice/.ssh/authorized_keys", + "manage_dir": true, + "path": null, + "state": "present", + "user": "alice", + "validate_certs": true +} + +TASK [user_provision : Get users to remove from lock file] ********************* +ok: [localhost] => { + "ansible_facts": { + "users_to_remove": [] + }, + "changed": false +} + +TASK [user_provision : Debug local user cleanup] ******************************* +ok: [localhost] => {} + +MSG: + +Local user cleanup: + - users_to_remove: [] + + +TASK [user_provision : Remove local user accounts] ***************************** +skipping: [localhost] => { + "changed": false, + "skipped_reason": "No items in the list" +} + +TASK [user_provision : Check if user exists] *********************************** +fatal: [localhost]: FAILED! => { + "changed": false +} + +MSG: + +One or more supplied key could not be found in the database. +...ignoring + +TASK [user_provision : Set user exists status for debug] *********************** +ok: [localhost] => { + "ansible_facts": { + "user_exists_status": false + }, + "changed": false +} + +TASK [user_provision : Set user exists status for debug (success case)] ******** +skipping: [localhost] => { + "changed": false, + "false_condition": "debug | default(false) and user_exists_check is not failed", + "skip_reason": "Conditional result was False" +} + +TASK [user_provision : Debug user provision for bob] *************************** +ok: [localhost] => {} + +MSG: + +Provisioning user bob: + - user_secrets_changed: True + - password_actually_changed: True + - password_update_mode: always + - user_exists: False + + +TASK [user_provision : Create VDI user bob] ************************************ +[DEPRECATION WARNING]: Encryption using the Python crypt module is deprecated. +The Python crypt module is deprecated and will be removed from Python 3.13. +Install the passlib library for continued encryption functionality. This +feature will be removed in version 2.17. Deprecation warnings can be disabled +by setting deprecation_warnings=False in ansible.cfg. +changed: [localhost] => { + "changed": true, + "comment": "", + "create_home": true, + "group": 1003, + "groups": "vdiusers,wheel", + "home": "/home/bob", + "name": "bob", + "password": "NOT_LOGGING_PASSWORD", + "shell": "/bin/bash", + "state": "present", + "system": false, + "uid": 1001 +} + +TASK [user_provision : Ensure .ssh directory for bob] ************************** +changed: [localhost] => { + "changed": true, + "gid": 1001, + "group": "vdiusers", + "mode": "0700", + "owner": "bob", + "path": "/home/bob/.ssh", + "size": 6, + "state": "directory", + "uid": 1001 +} + +TASK [user_provision : Generate SSH key pair for bob] ************************** +changed: [localhost] => { + "changed": true, + "comment": "", + "filename": "/home/bob/.ssh/id_rsa", + "fingerprint": "SHA256:lKDSzJMXof3DNSJMGYZaAIQO6AFMq0pI5o2to6VG4ic", + "public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCzDZtYeeQAFPzpE14NfR0OHKtADWTNEfulc02pYEGcm+fBptIrFd4IH4J8CF26rUGcdryQ+blRc4UtyUyrIZurK9Q+I8kIM0ggxnLmCM6rs2ycKAayJQihQI8wIa8mQr2OYea4g44fdZr79tZSpGkCmeT9zE432nPDtBxfXw9/zOm+d086lejm/CWxRFidGGiiodWLJR3Sg0VqChij9+TKfplAZCtfuNcKJGywEVkpvNBZ3webAmqDKwgN1Q5cM6H4DntC2FT9jBx6IqCOmXQzQSVjO5BGU1djBLWL/ANJQ9pTfiPxLee4Jl1rT89UhnnN4dJMCrKkEVz4Sf82MUU+S2mnfnlk6U3NSXPfsruy0puttwN0FPjWeq5f5vNCCwuI9PBbmbS4++QyqSgE5JN+C46SlOnxyMQxM0bp5VTmB8AyjXiN5JrTUHlf+H0JIAqspHMEAVaD8jiM0CZ4Oh7Z/t71ZGIMWvY/93IKk5uVEizpazBS42TgmbprjGMTbTLKi3w8oAAQulNR3pnfyt4XekFbmVXlKK12A1nmQgaGx2naUm1bmtUT4yiRb1VDHKHS8+SGjLkKzgud/tkodFMUUhbCNlTgRjhJ78VIkYCIDyymNANbd2cB4wk9tpw845mau47Lvw6tUl33Anh97HUCl5CUI/lcLgOAXpasdD1JnQ==", + "size": 4096, + "type": "rsa" +} + +TASK [user_provision : Add public key to authorized_keys for bob] ************** +changed: [localhost] => { + "changed": true, + "comment": null, + "exclusive": false, + "follow": false, + "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCzDZtYeeQAFPzpE14NfR0OHKtADWTNEfulc02pYEGcm+fBptIrFd4IH4J8CF26rUGcdryQ+blRc4UtyUyrIZurK9Q+I8kIM0ggxnLmCM6rs2ycKAayJQihQI8wIa8mQr2OYea4g44fdZr79tZSpGkCmeT9zE432nPDtBxfXw9/zOm+d086lejm/CWxRFidGGiiodWLJR3Sg0VqChij9+TKfplAZCtfuNcKJGywEVkpvNBZ3webAmqDKwgN1Q5cM6H4DntC2FT9jBx6IqCOmXQzQSVjO5BGU1djBLWL/ANJQ9pTfiPxLee4Jl1rT89UhnnN4dJMCrKkEVz4Sf82MUU+S2mnfnlk6U3NSXPfsruy0puttwN0FPjWeq5f5vNCCwuI9PBbmbS4++QyqSgE5JN+C46SlOnxyMQxM0bp5VTmB8AyjXiN5JrTUHlf+H0JIAqspHMEAVaD8jiM0CZ4Oh7Z/t71ZGIMWvY/93IKk5uVEizpazBS42TgmbprjGMTbTLKi3w8oAAQulNR3pnfyt4XekFbmVXlKK12A1nmQgaGx2naUm1bmtUT4yiRb1VDHKHS8+SGjLkKzgud/tkodFMUUhbCNlTgRjhJ78VIkYCIDyymNANbd2cB4wk9tpw845mau47Lvw6tUl33Anh97HUCl5CUI/lcLgOAXpasdD1JnQ==", + "key_options": null, + "keyfile": "/home/bob/.ssh/authorized_keys", + "manage_dir": true, + "path": null, + "state": "present", + "user": "bob", + "validate_certs": true +} + +TASK [user_provision : Get users to remove from lock file] ********************* +ok: [localhost] => { + "ansible_facts": { + "users_to_remove": [] + }, + "changed": false +} + +TASK [user_provision : Debug local user cleanup] ******************************* +ok: [localhost] => {} + +MSG: + +Local user cleanup: + - users_to_remove: [] + + +TASK [user_provision : Remove local user accounts] ***************************** +skipping: [localhost] => { + "changed": false, + "skipped_reason": "No items in the list" +} + +TASK [user_provision : Provision OS Login users] ******************************* +skipping: [localhost] => (item={'username': 'alice', 'port': 5901, 'password': 'xc61q4jU3mUCo84M', 'vncserver_password': '4JB1YHUQ', 'display_number': 1}) => { + "ansible_loop_var": "item", + "changed": false, + "false_condition": "user_provision | lower == 'os_login'", + "item": { + "display_number": 1, + "password": "xc61q4jU3mUCo84M", + "port": 5901, + "username": "alice", + "vncserver_password": "4JB1YHUQ" + }, + "skip_reason": "Conditional result was False" +} +skipping: [localhost] => (item={'username': 'bob', 'port': 5902, 'secret_name': 'a-password-for-bob', 'password': 'B0bp4ssw0rd123', 'vncserver_password': '7DmxvFp2', 'display_number': 2}) => { + "ansible_loop_var": "item", + "changed": false, + "false_condition": "user_provision | lower == 'os_login'", + "item": { + "display_number": 2, + "password": "B0bp4ssw0rd123", + "port": 5902, + "secret_name": "a-password-for-bob", + "username": "bob", + "vncserver_password": "7DmxvFp2" + }, + "skip_reason": "Conditional result was False" +} +skipping: [localhost] => { + "changed": false +} + +MSG: + +All items skipped + +TASK [user_provision : Set current role for lock manager completion] *********** +ok: [localhost] => { + "ansible_facts": { + "current_role": "user_provision", + "role_completed": true + }, + "changed": false +} + +TASK [lock_manager : Debug create lock operation] ****************************** +ok: [localhost] => {} + +MSG: + +Creating/updating lock file for role: user_provision + +TASK [lock_manager : Check if lock file exists] ******************************** +ok: [localhost] => { + "changed": false, + "stat": { + "atime": 1753371005.05256, + "attr_flags": "", + "attributes": [], + "block_size": 4096, + "blocks": 8, + "charset": "us-ascii", + "checksum": "9205f17d8ce64bf55a046c433f6f24c4efa12eae", + "ctime": 1753371004.8265433, + "dev": 2050, + "device_type": 0, + "executable": false, + "exists": true, + "gid": 0, + "gr_name": "root", + "inode": 51539232, + "isblk": false, + "ischr": false, + "isdir": false, + "isfifo": false, + "isgid": false, + "islnk": false, + "isreg": true, + "issock": false, + "isuid": false, + "mimetype": "text/plain", + "mode": "0644", + "mtime": 1753371004.6825325, + "nlink": 1, + "path": "/opt/vdi-setup/.vdi-lock.yaml", + "pw_name": "root", + "readable": true, + "rgrp": true, + "roth": true, + "rusr": true, + "size": 982, + "uid": 0, + "version": "2720298527", + "wgrp": false, + "woth": false, + "writeable": true, + "wusr": true, + "xgrp": false, + "xoth": false, + "xusr": false + } +} + +TASK [lock_manager : Debug lock file existence] ******************************** +ok: [localhost] => {} + +MSG: + +Lock file exists: True + +TASK [lock_manager : Create initial lock file structure] *********************** +skipping: [localhost] => { + "changed": false, + "false_condition": "not lock_file_stat.stat.exists", + "skip_reason": "Conditional result was False" +} + +TASK [lock_manager : Debug lock file creation] ********************************* +skipping: [localhost] => { + "false_condition": "debug | default(false) and not lock_file_stat.stat.exists" +} + +TASK [lock_manager : Load existing lock file data] ***************************** +ok: [localhost] => { + "ansible_facts": { + "vdi_setup_status": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:03Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:03Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "ansible_included_var_files": [ + "/opt/vdi-setup/.vdi-lock.yaml" + ], + "changed": false +} + +TASK [lock_manager : Get current lock data] ************************************ +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:03Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:03Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Update deployment hash] *********************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:03Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:03Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Get current timestamp] ************************************ +changed: [localhost] => { + "changed": true, + "cmd": [ + "date", + "-u", + "+%Y-%m-%dT%H:%M:%SZ" + ], + "delta": "0:00:00.002381", + "end": "2025-07-24 15:30:16.755905", + "rc": 0, + "start": "2025-07-24 15:30:16.753524" +} + +STDOUT: + +2025-07-24T15:30:16Z + +TASK [lock_manager : Update last updated timestamp] **************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:16Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:03Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Get current user secrets status] ************************** +ok: [localhost] => { + "ansible_facts": { + "current_user_secrets": { + "last_secret_check": "2025-07-24T15:30:03Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Create user secrets update] ******************************* +ok: [localhost] => { + "ansible_facts": { + "user_secrets_update": { + "last_secret_check": "2025-07-24T15:30:16Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" + } + }, + "changed": false +} + +TASK [lock_manager : Update user secrets hash] ********************************* +ok: [localhost] => { + "ansible_facts": { + "updated_user_secrets": { + "last_secret_check": "2025-07-24T15:30:16Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Apply user secrets update] ******************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:16Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:16Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Create role status entry] ********************************* +ok: [localhost] => { + "ansible_facts": { + "role_status_entry": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + } + }, + "changed": false +} + +TASK [lock_manager : Debug role status entry] ********************************** +ok: [localhost] => {} + +MSG: + +Role status entry for user_provision: + - completed: True + - completed_at: 2025-07-24T15:30:16Z + + +TASK [lock_manager : Debug current lock data before role update] *************** +ok: [localhost] => {} + +MSG: + +Current lock data before updating user_provision: + - current_lock_data: completed_roles: + base_os: + completed: true + completed_at: '2025-07-24T15:29:50Z' + lock_manager: + completed: true + completed_at: '2025-07-24T15:27:44Z' + secret_manager: + completed: true + completed_at: '2025-07-24T15:30:03Z' + user_provision: + completed: false + vdi_tool: + completed: false + vnc: + completed: false +created_at: '2025-07-24T15:27:41Z' +deployment_hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab +deployment_name: vdi-test-scott +force_rerun: false +last_updated: '2025-07-24T15:30:16Z' +lock_version: '1.0' +setup_status: configuring +user_management: + current_users: + - alice + - bob + previous_users: + - alice + - bob + removed_users: [] +user_secrets_status: + last_secret_check: '2025-07-24T15:30:16Z' + user_secrets_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 + users_updated: + - alice + - bob + + - current_completed_roles: base_os: + completed: true + completed_at: '2025-07-24T15:29:50Z' +lock_manager: + completed: true + completed_at: '2025-07-24T15:27:44Z' +secret_manager: + completed: true + completed_at: '2025-07-24T15:30:03Z' +user_provision: + completed: false +vdi_tool: + completed: false +vnc: + completed: false + + - role_completed: True + - current_role: user_provision + + +TASK [lock_manager : Get current completed roles] ****************************** +ok: [localhost] => { + "ansible_facts": { + "current_completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": false + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + } + }, + "changed": false +} + +TASK [lock_manager : Update completed roles] *********************************** +ok: [localhost] => { + "ansible_facts": { + "updated_completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + } + }, + "changed": false +} + +TASK [lock_manager : Update lock data with completed roles] ******************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:16Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:16Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Debug updated lock data after role update] **************** +ok: [localhost] => {} + +MSG: + +Updated lock data after updating user_provision: + - current_role: user_provision + - role_completed: True + - completed_roles: {'base_os': {'completed': True, 'completed_at': '2025-07-24T15:29:50Z'}, 'lock_manager': {'completed': True, 'completed_at': '2025-07-24T15:27:44Z'}, 'secret_manager': {'completed': True, 'completed_at': '2025-07-24T15:30:03Z'}, 'user_provision': {'completed': True, 'completed_at': '2025-07-24T15:30:16Z'}, 'vdi_tool': {'completed': False}, 'vnc': {'completed': False}} + + +TASK [lock_manager : Get current users updated list] *************************** +ok: [localhost] => { + "ansible_facts": { + "current_users_updated": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Get new usernames list] *********************************** +ok: [localhost] => { + "ansible_facts": { + "new_usernames": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Combine users updated lists] ****************************** +ok: [localhost] => { + "ansible_facts": { + "combined_users_updated": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Debug user deduplication] ********************************* +ok: [localhost] => {} + +MSG: + +User deduplication for user_provision: + - current_users_updated: ['alice', 'bob'] + - new_usernames: ['alice', 'bob'] + - combined_users_updated: ['alice', 'bob'] + + +TASK [lock_manager : Get current user secrets for list update] ***************** +ok: [localhost] => { + "ansible_facts": { + "current_user_secrets_for_list": { + "last_secret_check": "2025-07-24T15:30:16Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Update users updated list] ******************************** +ok: [localhost] => { + "ansible_facts": { + "updated_user_secrets_with_list": { + "last_secret_check": "2025-07-24T15:30:16Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Apply users updated list] ********************************* +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:16Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:16Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Get current user management data] ************************* +ok: [localhost] => { + "ansible_facts": { + "current_user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + } + }, + "changed": false +} + +TASK [lock_manager : Get current usernames list] ******************************* +ok: [localhost] => { + "ansible_facts": { + "current_usernames": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Update user management data] ****************************** +ok: [localhost] => { + "ansible_facts": { + "updated_user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + } + }, + "changed": false +} + +TASK [lock_manager : Apply user management update] ***************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:16Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:16Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Debug user management] ************************************ +ok: [localhost] => {} + +MSG: + +User management for user_provision: + - current_users: ['alice', 'bob'] + - previous_users: ['alice', 'bob'] + - removed_users: [] + + +TASK [lock_manager : Check if all roles are completed] ************************* +ok: [localhost] => { + "ansible_facts": { + "all_roles_completed": false + }, + "changed": false +} + +TASK [lock_manager : Update setup status based on completion] ****************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:16Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:16Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Debug setup status update] ******************************** +ok: [localhost] => {} + +MSG: + +Setup status update: + - all_roles_completed: False + - setup_status: configuring + - completed_roles_count: 4 + - total_roles_count: 6 + + +TASK [lock_manager : Write updated lock file] ********************************** +changed: [localhost] => { + "changed": true, + "checksum": "ca35350cf060981f0667398fcb15c8e12423a97e", + "dest": "/opt/vdi-setup/.vdi-lock.yaml", + "gid": 0, + "group": "root", + "md5sum": "7ed7d0c10ced955765662b1feee9212a", + "mode": "0644", + "owner": "root", + "size": 1024, + "src": "/root/.ansible/tmp/ansible-tmp-1753371017.8474565-25936-21002111182327/source", + "state": "file", + "uid": 0 +} + +TASK [lock_manager : Debug VM metadata update] ********************************* +ok: [localhost] => {} + +MSG: + +VM metadata update info: + - hostname: vdi-test-scott-rocky-0 + - zone: us-central1-a + - lock_file_changed: True + + +TASK [lock_manager : Get lock file content as base64] ************************** +changed: [localhost] => { + "changed": true, + "cmd": "cat /opt/vdi-setup/.vdi-lock.yaml | base64 -w 0", + "delta": "0:00:00.003777", + "end": "2025-07-24 15:30:18.389398", + "rc": 0, + "start": "2025-07-24 15:30:18.385621" +} + +STDOUT: + +dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiB0cnVlCiAgICAgIGNvbXBsZXRlZF9hdDogJzIwMjUtMDctMjRUMTU6MzA6MTZaJwogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogZmFsc2UKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiBmYWxzZQogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNTozMDoxNlonCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogY29uZmlndXJpbmcKICB1c2VyX21hbmFnZW1lbnQ6CiAgICBjdXJyZW50X3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHByZXZpb3VzX3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHJlbW92ZWRfdXNlcnM6IFtdCiAgdXNlcl9zZWNyZXRzX3N0YXR1czoKICAgIGxhc3Rfc2VjcmV0X2NoZWNrOiAnMjAyNS0wNy0yNFQxNTozMDoxNlonCiAgICB1c2VyX3NlY3JldHNfaGFzaDogNmE2YTAyZDNkMDk5MzNjYTI4MmFlMmIwMjMzNjM0M2M5ZDczNzAxZTQ1YmQ4MTQ3NWZiZDkyZGNmOWQyZjNiNgogICAgdXNlcnNfdXBkYXRlZDoKICAgIC0gYWxpY2UKICAgIC0gYm9iCg== + +TASK [lock_manager : Add VDI lock file content to VM metadata] ***************** +changed: [localhost] => { + "changed": true, + "cmd": [ + "gcloud", + "compute", + "instances", + "add-metadata", + "vdi-test-scott-rocky-0", + "--metadata", + "vdi-lock-content=dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiB0cnVlCiAgICAgIGNvbXBsZXRlZF9hdDogJzIwMjUtMDctMjRUMTU6MzA6MTZaJwogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogZmFsc2UKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiBmYWxzZQogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNTozMDoxNlonCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogY29uZmlndXJpbmcKICB1c2VyX21hbmFnZW1lbnQ6CiAgICBjdXJyZW50X3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHByZXZpb3VzX3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHJlbW92ZWRfdXNlcnM6IFtdCiAgdXNlcl9zZWNyZXRzX3N0YXR1czoKICAgIGxhc3Rfc2VjcmV0X2NoZWNrOiAnMjAyNS0wNy0yNFQxNTozMDoxNlonCiAgICB1c2VyX3NlY3JldHNfaGFzaDogNmE2YTAyZDNkMDk5MzNjYTI4MmFlMmIwMjMzNjM0M2M5ZDczNzAxZTQ1YmQ4MTQ3NWZiZDkyZGNmOWQyZjNiNgogICAgdXNlcnNfdXBkYXRlZDoKICAgIC0gYWxpY2UKICAgIC0gYm9iCg==", + "--zone=us-central1-a" + ], + "delta": "0:00:04.229183", + "end": "2025-07-24 15:30:22.808719", + "failed_when_result": false, + "rc": 0, + "start": "2025-07-24 15:30:18.579536" +} + +STDERR: + +Updated [https://www.googleapis.com/compute/v1/projects/hpc-discovery-external/zones/us-central1-a/instances/vdi-test-scott-rocky-0]. + +TASK [lock_manager : Debug lock file write] ************************************ +ok: [localhost] => {} + +MSG: + +Lock file written: True + +TASK [lock_manager : Verify lock file integrity] ******************************* +ok: [localhost] => { + "changed": false, + "cmd": [ + "python3", + "-c", + "import yaml; yaml.safe_load(open('/opt/vdi-setup/.vdi-lock.yaml'))" + ], + "delta": "0:00:00.040331", + "end": "2025-07-24 15:30:23.092904", + "rc": 0, + "start": "2025-07-24 15:30:23.052573" +} + +TASK [lock_manager : Debug lock file validation] ******************************* +ok: [localhost] => {} + +MSG: + +Lock file validation: True + +TASK [lock_manager : Check if VDI monitor service exists] ********************** +ok: [localhost] => { + "changed": false, + "stat": { + "atime": 1753370874.1548011, + "attr_flags": "", + "attributes": [], + "block_size": 4096, + "blocks": 8, + "charset": "us-ascii", + "checksum": "f6de22707f8d9248014f679a2f1930451e2bf635", + "ctime": 1753370872.6256874, + "dev": 2050, + "device_type": 0, + "executable": false, + "exists": true, + "gid": 0, + "gr_name": "root", + "inode": 285214140, + "isblk": false, + "ischr": false, + "isdir": false, + "isfifo": false, + "isgid": false, + "islnk": false, + "isreg": true, + "issock": false, + "isuid": false, + "mimetype": "text/plain", + "mode": "0644", + "mtime": 1753370872.4766762, + "nlink": 1, + "path": "/etc/systemd/system/vdi-monitor.service", + "pw_name": "root", + "readable": true, + "rgrp": true, + "roth": true, + "rusr": true, + "size": 283, + "uid": 0, + "version": "1338188335", + "wgrp": false, + "woth": false, + "writeable": true, + "wusr": true, + "xgrp": false, + "xoth": false, + "xusr": false + } +} + +TASK [lock_manager : Debug VDI monitor setup conditions] *********************** +ok: [localhost] => {} + +MSG: + +VDI monitor setup conditions: + - current_role: user_provision + - vdi_monitor_service_exists: True + - lock_file_creation.changed: False + - will_setup_monitor: False + + +TASK [lock_manager : Setup VDI monitoring service (first time only)] *********** +skipping: [localhost] => { + "changed": false, + "false_condition": "not vdi_monitor_service_stat.stat.exists", + "skip_reason": "Conditional result was False" +} + +TASK [lock_manager : Fail if lock file is invalid] ***************************** +skipping: [localhost] => { + "changed": false, + "false_condition": "lock_file_validation.rc != 0", + "skip_reason": "Conditional result was False" +} + +TASK [vnc : Set current role for lock manager check] *************************** +ok: [localhost] => { + "ansible_facts": { + "current_role": "vnc" + }, + "changed": false +} + +TASK [lock_manager : Check if lock file exists] ******************************** +ok: [localhost] => { + "changed": false, + "stat": { + "atime": 1753371018.3885562, + "attr_flags": "", + "attributes": [], + "block_size": 4096, + "blocks": 8, + "charset": "us-ascii", + "checksum": "ca35350cf060981f0667398fcb15c8e12423a97e", + "ctime": 1753371018.1625395, + "dev": 2050, + "device_type": 0, + "executable": false, + "exists": true, + "gid": 0, + "gr_name": "root", + "inode": 117754419, + "isblk": false, + "ischr": false, + "isdir": false, + "isfifo": false, + "isgid": false, + "islnk": false, + "isreg": true, + "issock": false, + "isuid": false, + "mimetype": "text/plain", + "mode": "0644", + "mtime": 1753371018.0155284, + "nlink": 1, + "path": "/opt/vdi-setup/.vdi-lock.yaml", + "pw_name": "root", + "readable": true, + "rgrp": true, + "roth": true, + "rusr": true, + "size": 1024, + "uid": 0, + "version": "3086664638", + "wgrp": false, + "woth": false, + "writeable": true, + "wusr": true, + "xgrp": false, + "xoth": false, + "xusr": false + } +} + +TASK [lock_manager : Debug lock file existence] ******************************** +ok: [localhost] => {} + +MSG: + +Lock file exists: True + +TASK [lock_manager : Load existing lock file] ********************************** +ok: [localhost] => { + "ansible_facts": { + "vdi_setup_status": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:16Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:16Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "ansible_included_var_files": [ + "/opt/vdi-setup/.vdi-lock.yaml" + ], + "changed": false +} + +TASK [lock_manager : Debug lock file load result] ****************************** +ok: [localhost] => {} + +MSG: + +Lock file loaded successfully: True + +TASK [lock_manager : Calculate current deployment hash] ************************ +ok: [localhost] => { + "ansible_facts": { + "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab" + }, + "changed": false +} + +TASK [lock_manager : Debug deployment hash calculation] ************************ +ok: [localhost] => {} + +MSG: + +Current deployment hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab +Deployment components: + - deployment_name: vdi-test-scott + - project_id: hpc-discovery-external + - vdi_tool: guacamole + - vnc_flavor: tigervnc + - user_provision: local_users + - vdi_webapp_port: 8081 + - vdi_resolution: 1920x1080 + - vdi_users count: 2 + + +TASK [lock_manager : Calculate current user secrets hash] ********************** +ok: [localhost] => { + "ansible_facts": { + "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" + }, + "changed": false +} + +TASK [lock_manager : Debug user secrets hash calculation] ********************** +ok: [localhost] => {} + +MSG: + +Current user secrets hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 +User configuration components: + - vdi_users (blueprint): [{"username": "alice", "port": 5901}, {"username": "bob", "port": 5902, "secret_name": "a-password-for-bob"}] + + +TASK [lock_manager : Set default values for missing lock file] ***************** +skipping: [localhost] => { + "changed": false, + "false_condition": "not lock_file_stat.stat.exists", + "skip_reason": "Conditional result was False" +} + +TASK [lock_manager : Debug lock file status] *********************************** +ok: [localhost] => {} + +MSG: + +Lock file status: + - exists: True + - deployment_hash: none + - user_secrets_hash: none + - force_rerun: False + + +TASK [lock_manager : Check if lock file exists] ******************************** +ok: [localhost] => { + "ansible_facts": { + "lock_file_exists": true + }, + "changed": false +} + +TASK [lock_manager : Check if current role is completed] *********************** +ok: [localhost] => { + "ansible_facts": { + "current_role_completed": false + }, + "changed": false +} + +TASK [lock_manager : Check if deployment hash matches] ************************* +ok: [localhost] => { + "ansible_facts": { + "deployment_hash_matches": false + }, + "changed": false +} + +TASK [lock_manager : Check if force rerun is enabled] ************************** +ok: [localhost] => { + "ansible_facts": { + "force_rerun_enabled": false + }, + "changed": false +} + +TASK [lock_manager : Determine if role should run] ***************************** +ok: [localhost] => { + "ansible_facts": { + "role_should_run": true + }, + "changed": false +} + +TASK [lock_manager : Debug role should run calculation] ************************ +ok: [localhost] => {} + +MSG: + +Role should run calculation for vnc: + - not lock_file_exists: False + - not current_role_completed: True + - not deployment_hash_matches: True + - force_rerun_enabled: False + - role_should_run: True + + +TASK [lock_manager : Debug role execution decision] **************************** +ok: [localhost] => {} + +MSG: + +Role execution decision for vnc: + - current_role: vnc + - ansible_role_name: lock_manager + - role_should_run: True + - lock_file_exists: True + - current_role_completed: False + - deployment_hash_matches: False + - force_rerun_enabled: False + + +TASK [lock_manager : Check if user configuration has changed] ****************** +ok: [localhost] => { + "ansible_facts": { + "user_config_changed": true + }, + "changed": false +} + +TASK [lock_manager : Set user secrets changed flag (will be updated by secret_manager role)] *** +ok: [localhost] => { + "ansible_facts": { + "user_secrets_changed": true + }, + "changed": false +} + +TASK [lock_manager : Debug user secrets change detection] ********************** +ok: [localhost] => {} + +MSG: + +User configuration change detection: + - user_config_changed: True + - user_secrets_changed: True + - stored_hash: none + - current_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 + + +TASK [lock_manager : Set lock check result] ************************************ +ok: [localhost] => { + "ansible_facts": { + "lock_check_result": { + "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "existing_deployment_hash": "none", + "existing_user_secrets_hash": "none", + "force_rerun": false, + "role_should_run": true, + "user_secrets_changed": true + } + }, + "changed": false +} + +TASK [lock_manager : Debug final lock check result] **************************** +ok: [localhost] => {} + +MSG: + +Final lock check result: + - role_should_run: True + - deployment_hash_changed: True + - user_secrets_changed: True + - force_rerun: False + + +TASK [vnc : Set role_should_run from lock check result] ************************ +ok: [localhost] => { + "ansible_facts": { + "role_should_run": true + }, + "changed": false +} + +TASK [vnc : Skip vnc tasks if role should not run] ***************************** +skipping: [localhost] => { + "false_condition": "not role_should_run" +} + +TASK [vnc : Install TigerVNC] ************************************************** +included: /tmp/vdi/roles/vnc/tasks/tigervnc.yaml for localhost + +TASK [vnc : Configure per-user VNC settings] *********************************** +included: /tmp/vdi/roles/vnc/tasks/vnc_user_config.yaml for localhost => (item={'username': 'alice', 'port': 5901, 'password': 'xc61q4jU3mUCo84M', 'vncserver_password': '4JB1YHUQ', 'display_number': 1}) +included: /tmp/vdi/roles/vnc/tasks/vnc_user_config.yaml for localhost => (item={'username': 'bob', 'port': 5902, 'secret_name': 'a-password-for-bob', 'password': 'B0bp4ssw0rd123', 'vncserver_password': '7DmxvFp2', 'display_number': 2}) + +TASK [vnc : Ensure VNC configuration directory exists] ************************* +changed: [localhost] => { + "changed": true, + "gid": 1001, + "group": "vdiusers", + "mode": "0700", + "owner": "alice", + "path": "/home/alice/.vnc", + "size": 6, + "state": "directory", + "uid": 1000 +} + +TASK [vnc : Write VNC xstartup for alice (Ubuntu/Debian)] ********************** +skipping: [localhost] => { + "changed": false, + "false_condition": "ansible_distribution in [\"Ubuntu\", \"Debian\"]", + "skip_reason": "Conditional result was False" +} + +TASK [vnc : Write VNC password file for alice] ********************************* +changed: [localhost] => { + "changed": true, + "cmd": "echo \"4JB1YHUQ\" | vncpasswd -f > \"/home/alice/.vnc/passwd\"\n", + "delta": "0:00:00.009600", + "end": "2025-07-24 15:30:24.957369", + "rc": 0, + "start": "2025-07-24 15:30:24.947769" +} + +TASK [vnc : Fix passwd file permissions] *************************************** +changed: [localhost] => { + "changed": true, + "gid": 1001, + "group": "vdiusers", + "mode": "0600", + "owner": "alice", + "path": "/home/alice/.vnc/passwd", + "size": 8, + "state": "file", + "uid": 1000 +} + +TASK [vnc : Copy per-user VNC config] ****************************************** +changed: [localhost] => { + "changed": true, + "checksum": "e5a3133663f35d3a79dc06917090845ab424c412", + "dest": "/home/alice/.vnc/config", + "gid": 1001, + "group": "vdiusers", + "md5sum": "15686602524e414a443391251ac46b4a", + "mode": "0644", + "owner": "alice", + "size": 32, + "src": "/root/.ansible/tmp/ansible-tmp-1753371025.2160182-26228-110817536086157/source", + "state": "file", + "uid": 1000 +} + +TASK [vnc : Ensure VNC configuration directory exists] ************************* +changed: [localhost] => { + "changed": true, + "gid": 1001, + "group": "vdiusers", + "mode": "0700", + "owner": "bob", + "path": "/home/bob/.vnc", + "size": 6, + "state": "directory", + "uid": 1001 +} + +TASK [vnc : Write VNC xstartup for bob (Ubuntu/Debian)] ************************ +skipping: [localhost] => { + "changed": false, + "false_condition": "ansible_distribution in [\"Ubuntu\", \"Debian\"]", + "skip_reason": "Conditional result was False" +} + +TASK [vnc : Write VNC password file for bob] *********************************** +changed: [localhost] => { + "changed": true, + "cmd": "echo \"7DmxvFp2\" | vncpasswd -f > \"/home/bob/.vnc/passwd\"\n", + "delta": "0:00:00.007207", + "end": "2025-07-24 15:30:26.051034", + "rc": 0, + "start": "2025-07-24 15:30:26.043827" +} + +TASK [vnc : Fix passwd file permissions] *************************************** +changed: [localhost] => { + "changed": true, + "gid": 1001, + "group": "vdiusers", + "mode": "0600", + "owner": "bob", + "path": "/home/bob/.vnc/passwd", + "size": 8, + "state": "file", + "uid": 1001 +} + +TASK [vnc : Copy per-user VNC config] ****************************************** +changed: [localhost] => { + "changed": true, + "checksum": "e5a3133663f35d3a79dc06917090845ab424c412", + "dest": "/home/bob/.vnc/config", + "gid": 1001, + "group": "vdiusers", + "md5sum": "15686602524e414a443391251ac46b4a", + "mode": "0644", + "owner": "bob", + "size": 32, + "src": "/root/.ansible/tmp/ansible-tmp-1753371026.302065-26344-216915276880984/source", + "state": "file", + "uid": 1001 +} + +TASK [vnc : Template TigerVNC users file] ************************************** +changed: [localhost] => { + "changed": true, + "checksum": "07719f46faaccbdf742785d5d2310a55a7352cdb", + "dest": "/etc/tigervnc/vncserver.users", + "gid": 0, + "group": "root", + "md5sum": "4e1b63b33d08dda6c1a7dde083fe9854", + "mode": "0644", + "owner": "root", + "size": 16, + "src": "/root/.ansible/tmp/ansible-tmp-1753371026.663593-26366-244603176491990/source", + "state": "file", + "uid": 0 +} + +TASK [vnc : Deploy per-user TigerVNC systemd unit (Ubuntu/Debian)] ************* +skipping: [localhost] => (item={'username': 'alice', 'port': 5901, 'password': 'xc61q4jU3mUCo84M', 'vncserver_password': '4JB1YHUQ', 'display_number': 1}) => { + "ansible_loop_var": "item", + "changed": false, + "false_condition": "ansible_distribution in [\"Ubuntu\", \"Debian\"]", + "item": { + "display_number": 1, + "password": "xc61q4jU3mUCo84M", + "port": 5901, + "username": "alice", + "vncserver_password": "4JB1YHUQ" + }, + "skip_reason": "Conditional result was False" +} +skipping: [localhost] => (item={'username': 'bob', 'port': 5902, 'secret_name': 'a-password-for-bob', 'password': 'B0bp4ssw0rd123', 'vncserver_password': '7DmxvFp2', 'display_number': 2}) => { + "ansible_loop_var": "item", + "changed": false, + "false_condition": "ansible_distribution in [\"Ubuntu\", \"Debian\"]", + "item": { + "display_number": 2, + "password": "B0bp4ssw0rd123", + "port": 5902, + "secret_name": "a-password-for-bob", + "username": "bob", + "vncserver_password": "7DmxvFp2" + }, + "skip_reason": "Conditional result was False" +} +skipping: [localhost] => { + "changed": false +} + +MSG: + +All items skipped + +TASK [vnc : Deploy TigerVNC systemd unit (Rocky)] ****************************** +changed: [localhost] => { + "changed": true, + "checksum": "bd078a72af7384ad8cc0fdd595ea34d71f75969c", + "dest": "/etc/systemd/system/vncserver@.service", + "gid": 0, + "group": "root", + "md5sum": "a6a8e5fa35e07463b97f8750bab854fa", + "mode": "0644", + "owner": "root", + "size": 268, + "src": "/root/.ansible/tmp/ansible-tmp-1753371027.0719545-26392-180913534199741/source", + "state": "file", + "uid": 0 +} + +TASK [vnc : Reload systemd daemon] ********************************************* +ok: [localhost] => { + "changed": false, + "name": null, + "status": {} +} + +TASK [vnc : Enable & start TigerVNC (Ubuntu/Debian)] *************************** +skipping: [localhost] => (item={'username': 'alice', 'port': 5901, 'password': 'xc61q4jU3mUCo84M', 'vncserver_password': '4JB1YHUQ', 'display_number': 1}) => { + "ansible_loop_var": "item", + "changed": false, + "false_condition": "ansible_distribution in [\"Ubuntu\", \"Debian\"]", + "item": { + "display_number": 1, + "password": "xc61q4jU3mUCo84M", + "port": 5901, + "username": "alice", + "vncserver_password": "4JB1YHUQ" + }, + "skip_reason": "Conditional result was False" +} +skipping: [localhost] => (item={'username': 'bob', 'port': 5902, 'secret_name': 'a-password-for-bob', 'password': 'B0bp4ssw0rd123', 'vncserver_password': '7DmxvFp2', 'display_number': 2}) => { + "ansible_loop_var": "item", + "changed": false, + "false_condition": "ansible_distribution in [\"Ubuntu\", \"Debian\"]", + "item": { + "display_number": 2, + "password": "B0bp4ssw0rd123", + "port": 5902, + "secret_name": "a-password-for-bob", + "username": "bob", + "vncserver_password": "7DmxvFp2" + }, + "skip_reason": "Conditional result was False" +} +skipping: [localhost] => { + "changed": false +} + +MSG: + +All items skipped + +TASK [vnc : Enable & start TigerVNC (Rocky)] *********************************** +changed: [localhost] => (item={'username': 'alice', 'port': 5901, 'password': 'xc61q4jU3mUCo84M', 'vncserver_password': '4JB1YHUQ', 'display_number': 1}) => { + "ansible_loop_var": "item", + "changed": true, + "enabled": true, + "item": { + "display_number": 1, + "password": "xc61q4jU3mUCo84M", + "port": 5901, + "username": "alice", + "vncserver_password": "4JB1YHUQ" + }, + "name": "vncserver@:1", + "state": "started", + "status": { + "ActiveEnterTimestampMonotonic": "0", + "ActiveExitTimestampMonotonic": "0", + "ActiveState": "inactive", + "After": "syslog.target basic.target sysinit.target system-vncserver.slice network.target systemd-journald.socket", + "AllowIsolate": "no", + "AllowedCPUs": "", + "AllowedMemoryNodes": "", + "AmbientCapabilities": "", + "AssertResult": "no", + "AssertTimestampMonotonic": "0", + "Before": "shutdown.target", + "BlockIOAccounting": "no", + "BlockIOWeight": "[not set]", + "CPUAccounting": "no", + "CPUAffinity": "", + "CPUAffinityFromNUMA": "no", + "CPUQuotaPerSecUSec": "infinity", + "CPUQuotaPeriodUSec": "infinity", + "CPUSchedulingPolicy": "0", + "CPUSchedulingPriority": "0", + "CPUSchedulingResetOnFork": "no", + "CPUShares": "[not set]", + "CPUUsageNSec": "[not set]", + "CPUWeight": "[not set]", + "CacheDirectoryMode": "0755", + "CanFreeze": "yes", + "CanIsolate": "no", + "CanReload": "no", + "CanStart": "yes", + "CanStop": "yes", + "CapabilityBoundingSet": "cap_chown cap_dac_override cap_dac_read_search cap_fowner cap_fsetid cap_kill cap_setgid cap_setuid cap_setpcap cap_linux_immutable cap_net_bind_service cap_net_broadcast cap_net_admin cap_net_raw cap_ipc_lock cap_ipc_owner cap_sys_module cap_sys_rawio cap_sys_chroot cap_sys_ptrace cap_sys_pacct cap_sys_admin cap_sys_boot cap_sys_nice cap_sys_resource cap_sys_time cap_sys_tty_config cap_mknod cap_lease cap_audit_write cap_audit_control cap_setfcap cap_mac_override cap_mac_admin cap_syslog cap_wake_alarm cap_block_suspend cap_audit_read cap_perfmon cap_bpf", + "CollectMode": "inactive", + "ConditionResult": "no", + "ConditionTimestampMonotonic": "0", + "ConfigurationDirectoryMode": "0755", + "Conflicts": "shutdown.target", + "ControlPID": "0", + "DefaultDependencies": "yes", + "DefaultMemoryLow": "0", + "DefaultMemoryMin": "0", + "Delegate": "no", + "Description": "TigerVNC server for user :1", + "DevicePolicy": "auto", + "DynamicUser": "no", + "EffectiveCPUs": "", + "EffectiveMemoryNodes": "", + "ExecMainCode": "0", + "ExecMainExitTimestampMonotonic": "0", + "ExecMainPID": "0", + "ExecMainStartTimestampMonotonic": "0", + "ExecMainStatus": "0", + "ExecStart": "{ path=/usr/libexec/vncsession-start ; argv[]=/usr/libexec/vncsession-start :1 ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", + "ExecStop": "{ path=/bin/true ; argv[]=/bin/true ; ignore_errors=yes ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", + "FailureAction": "none", + "FileDescriptorStoreMax": "0", + "FragmentPath": "/etc/systemd/system/vncserver@.service", + "FreezerState": "running", + "GID": "[not set]", + "GuessMainPID": "yes", + "IOAccounting": "no", + "IOSchedulingClass": "0", + "IOSchedulingPriority": "0", + "IOWeight": "[not set]", + "IPAccounting": "no", + "IPEgressBytes": "18446744073709551615", + "IPEgressPackets": "18446744073709551615", + "IPIngressBytes": "18446744073709551615", + "IPIngressPackets": "18446744073709551615", + "Id": "vncserver@:1.service", + "IgnoreOnIsolate": "no", + "IgnoreSIGPIPE": "yes", + "InactiveEnterTimestampMonotonic": "0", + "InactiveExitTimestampMonotonic": "0", + "JobRunningTimeoutUSec": "infinity", + "JobTimeoutAction": "none", + "JobTimeoutUSec": "infinity", + "KeyringMode": "private", + "KillMode": "control-group", + "KillSignal": "15", + "LimitAS": "infinity", + "LimitASSoft": "infinity", + "LimitCORE": "infinity", + "LimitCORESoft": "0", + "LimitCPU": "infinity", + "LimitCPUSoft": "infinity", + "LimitDATA": "infinity", + "LimitDATASoft": "infinity", + "LimitFSIZE": "infinity", + "LimitFSIZESoft": "infinity", + "LimitLOCKS": "infinity", + "LimitLOCKSSoft": "infinity", + "LimitMEMLOCK": "65536", + "LimitMEMLOCKSoft": "65536", + "LimitMSGQUEUE": "819200", + "LimitMSGQUEUESoft": "819200", + "LimitNICE": "0", + "LimitNICESoft": "0", + "LimitNOFILE": "262144", + "LimitNOFILESoft": "1024", + "LimitNPROC": "127383", + "LimitNPROCSoft": "127383", + "LimitRSS": "infinity", + "LimitRSSSoft": "infinity", + "LimitRTPRIO": "0", + "LimitRTPRIOSoft": "0", + "LimitRTTIME": "infinity", + "LimitRTTIMESoft": "infinity", + "LimitSIGPENDING": "127383", + "LimitSIGPENDINGSoft": "127383", + "LimitSTACK": "infinity", + "LimitSTACKSoft": "8388608", + "LoadState": "loaded", + "LockPersonality": "no", + "LogLevelMax": "-1", + "LogRateLimitBurst": "0", + "LogRateLimitIntervalUSec": "0", + "LogsDirectoryMode": "0755", + "MainPID": "0", + "MemoryAccounting": "yes", + "MemoryCurrent": "[not set]", + "MemoryDenyWriteExecute": "no", + "MemoryHigh": "infinity", + "MemoryLimit": "infinity", + "MemoryLow": "0", + "MemoryMax": "infinity", + "MemoryMin": "0", + "MemorySwapMax": "infinity", + "MountAPIVFS": "no", + "MountFlags": "", + "NFileDescriptorStore": "0", + "NRestarts": "0", + "NUMAMask": "", + "NUMAPolicy": "n/a", + "Names": "vncserver@:1.service", + "NeedDaemonReload": "no", + "Nice": "0", + "NoNewPrivileges": "no", + "NonBlocking": "no", + "NotifyAccess": "none", + "OOMScoreAdjust": "0", + "OnFailureJobMode": "replace", + "PIDFile": "/run/vncsession-:1.pid", + "PermissionsStartOnly": "no", + "Perpetual": "no", + "PrivateDevices": "no", + "PrivateMounts": "no", + "PrivateNetwork": "no", + "PrivateTmp": "no", + "PrivateUsers": "no", + "ProtectControlGroups": "no", + "ProtectHome": "no", + "ProtectKernelModules": "no", + "ProtectKernelTunables": "no", + "ProtectSystem": "no", + "RefuseManualStart": "no", + "RefuseManualStop": "no", + "RemainAfterExit": "no", + "RemoveIPC": "no", + "Requires": "sysinit.target system-vncserver.slice", + "Restart": "always", + "RestartUSec": "15s", + "RestrictNamespaces": "no", + "RestrictRealtime": "no", + "RestrictSUIDSGID": "no", + "Result": "success", + "RootDirectoryStartOnly": "no", + "RuntimeDirectoryMode": "0755", + "RuntimeDirectoryPreserve": "no", + "RuntimeMaxUSec": "infinity", + "SameProcessGroup": "no", + "SecureBits": "0", + "SendSIGHUP": "no", + "SendSIGKILL": "yes", + "Slice": "system-vncserver.slice", + "StandardError": "inherit", + "StandardInput": "null", + "StandardInputData": "", + "StandardOutput": "journal", + "StartLimitAction": "none", + "StartLimitBurst": "5", + "StartLimitIntervalUSec": "10s", + "StartupBlockIOWeight": "[not set]", + "StartupCPUShares": "[not set]", + "StartupCPUWeight": "[not set]", + "StartupIOWeight": "[not set]", + "StateChangeTimestampMonotonic": "0", + "StateDirectoryMode": "0755", + "StatusErrno": "0", + "StopWhenUnneeded": "no", + "SubState": "dead", + "SuccessAction": "none", + "SyslogFacility": "3", + "SyslogLevel": "6", + "SyslogLevelPrefix": "yes", + "SyslogPriority": "30", + "SystemCallErrorNumber": "0", + "TTYReset": "no", + "TTYVHangup": "no", + "TTYVTDisallocate": "no", + "TasksAccounting": "yes", + "TasksCurrent": "[not set]", + "TasksMax": "203813", + "TimeoutStartUSec": "1min 30s", + "TimeoutStopUSec": "1min 30s", + "TimerSlackNSec": "50000", + "Transient": "no", + "Type": "forking", + "UID": "[not set]", + "UMask": "0022", + "UnitFilePreset": "disabled", + "UnitFileState": "disabled", + "UtmpMode": "init", + "WatchdogTimestampMonotonic": "0", + "WatchdogUSec": "0" + } +} +changed: [localhost] => (item={'username': 'bob', 'port': 5902, 'secret_name': 'a-password-for-bob', 'password': 'B0bp4ssw0rd123', 'vncserver_password': '7DmxvFp2', 'display_number': 2}) => { + "ansible_loop_var": "item", + "changed": true, + "enabled": true, + "item": { + "display_number": 2, + "password": "B0bp4ssw0rd123", + "port": 5902, + "secret_name": "a-password-for-bob", + "username": "bob", + "vncserver_password": "7DmxvFp2" + }, + "name": "vncserver@:2", + "state": "started", + "status": { + "ActiveEnterTimestampMonotonic": "0", + "ActiveExitTimestampMonotonic": "0", + "ActiveState": "inactive", + "After": "syslog.target sysinit.target basic.target system-vncserver.slice systemd-journald.socket network.target", + "AllowIsolate": "no", + "AllowedCPUs": "", + "AllowedMemoryNodes": "", + "AmbientCapabilities": "", + "AssertResult": "no", + "AssertTimestampMonotonic": "0", + "Before": "shutdown.target", + "BlockIOAccounting": "no", + "BlockIOWeight": "[not set]", + "CPUAccounting": "no", + "CPUAffinity": "", + "CPUAffinityFromNUMA": "no", + "CPUQuotaPerSecUSec": "infinity", + "CPUQuotaPeriodUSec": "infinity", + "CPUSchedulingPolicy": "0", + "CPUSchedulingPriority": "0", + "CPUSchedulingResetOnFork": "no", + "CPUShares": "[not set]", + "CPUUsageNSec": "[not set]", + "CPUWeight": "[not set]", + "CacheDirectoryMode": "0755", + "CanFreeze": "yes", + "CanIsolate": "no", + "CanReload": "no", + "CanStart": "yes", + "CanStop": "yes", + "CapabilityBoundingSet": "cap_chown cap_dac_override cap_dac_read_search cap_fowner cap_fsetid cap_kill cap_setgid cap_setuid cap_setpcap cap_linux_immutable cap_net_bind_service cap_net_broadcast cap_net_admin cap_net_raw cap_ipc_lock cap_ipc_owner cap_sys_module cap_sys_rawio cap_sys_chroot cap_sys_ptrace cap_sys_pacct cap_sys_admin cap_sys_boot cap_sys_nice cap_sys_resource cap_sys_time cap_sys_tty_config cap_mknod cap_lease cap_audit_write cap_audit_control cap_setfcap cap_mac_override cap_mac_admin cap_syslog cap_wake_alarm cap_block_suspend cap_audit_read cap_perfmon cap_bpf", + "CollectMode": "inactive", + "ConditionResult": "no", + "ConditionTimestampMonotonic": "0", + "ConfigurationDirectoryMode": "0755", + "Conflicts": "shutdown.target", + "ControlPID": "0", + "DefaultDependencies": "yes", + "DefaultMemoryLow": "0", + "DefaultMemoryMin": "0", + "Delegate": "no", + "Description": "TigerVNC server for user :2", + "DevicePolicy": "auto", + "DynamicUser": "no", + "EffectiveCPUs": "", + "EffectiveMemoryNodes": "", + "ExecMainCode": "0", + "ExecMainExitTimestampMonotonic": "0", + "ExecMainPID": "0", + "ExecMainStartTimestampMonotonic": "0", + "ExecMainStatus": "0", + "ExecStart": "{ path=/usr/libexec/vncsession-start ; argv[]=/usr/libexec/vncsession-start :2 ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", + "ExecStop": "{ path=/bin/true ; argv[]=/bin/true ; ignore_errors=yes ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", + "FailureAction": "none", + "FileDescriptorStoreMax": "0", + "FragmentPath": "/etc/systemd/system/vncserver@.service", + "FreezerState": "running", + "GID": "[not set]", + "GuessMainPID": "yes", + "IOAccounting": "no", + "IOSchedulingClass": "0", + "IOSchedulingPriority": "0", + "IOWeight": "[not set]", + "IPAccounting": "no", + "IPEgressBytes": "18446744073709551615", + "IPEgressPackets": "18446744073709551615", + "IPIngressBytes": "18446744073709551615", + "IPIngressPackets": "18446744073709551615", + "Id": "vncserver@:2.service", + "IgnoreOnIsolate": "no", + "IgnoreSIGPIPE": "yes", + "InactiveEnterTimestampMonotonic": "0", + "InactiveExitTimestampMonotonic": "0", + "JobRunningTimeoutUSec": "infinity", + "JobTimeoutAction": "none", + "JobTimeoutUSec": "infinity", + "KeyringMode": "private", + "KillMode": "control-group", + "KillSignal": "15", + "LimitAS": "infinity", + "LimitASSoft": "infinity", + "LimitCORE": "infinity", + "LimitCORESoft": "0", + "LimitCPU": "infinity", + "LimitCPUSoft": "infinity", + "LimitDATA": "infinity", + "LimitDATASoft": "infinity", + "LimitFSIZE": "infinity", + "LimitFSIZESoft": "infinity", + "LimitLOCKS": "infinity", + "LimitLOCKSSoft": "infinity", + "LimitMEMLOCK": "65536", + "LimitMEMLOCKSoft": "65536", + "LimitMSGQUEUE": "819200", + "LimitMSGQUEUESoft": "819200", + "LimitNICE": "0", + "LimitNICESoft": "0", + "LimitNOFILE": "262144", + "LimitNOFILESoft": "1024", + "LimitNPROC": "127383", + "LimitNPROCSoft": "127383", + "LimitRSS": "infinity", + "LimitRSSSoft": "infinity", + "LimitRTPRIO": "0", + "LimitRTPRIOSoft": "0", + "LimitRTTIME": "infinity", + "LimitRTTIMESoft": "infinity", + "LimitSIGPENDING": "127383", + "LimitSIGPENDINGSoft": "127383", + "LimitSTACK": "infinity", + "LimitSTACKSoft": "8388608", + "LoadState": "loaded", + "LockPersonality": "no", + "LogLevelMax": "-1", + "LogRateLimitBurst": "0", + "LogRateLimitIntervalUSec": "0", + "LogsDirectoryMode": "0755", + "MainPID": "0", + "MemoryAccounting": "yes", + "MemoryCurrent": "[not set]", + "MemoryDenyWriteExecute": "no", + "MemoryHigh": "infinity", + "MemoryLimit": "infinity", + "MemoryLow": "0", + "MemoryMax": "infinity", + "MemoryMin": "0", + "MemorySwapMax": "infinity", + "MountAPIVFS": "no", + "MountFlags": "", + "NFileDescriptorStore": "0", + "NRestarts": "0", + "NUMAMask": "", + "NUMAPolicy": "n/a", + "Names": "vncserver@:2.service", + "NeedDaemonReload": "no", + "Nice": "0", + "NoNewPrivileges": "no", + "NonBlocking": "no", + "NotifyAccess": "none", + "OOMScoreAdjust": "0", + "OnFailureJobMode": "replace", + "PIDFile": "/run/vncsession-:2.pid", + "PermissionsStartOnly": "no", + "Perpetual": "no", + "PrivateDevices": "no", + "PrivateMounts": "no", + "PrivateNetwork": "no", + "PrivateTmp": "no", + "PrivateUsers": "no", + "ProtectControlGroups": "no", + "ProtectHome": "no", + "ProtectKernelModules": "no", + "ProtectKernelTunables": "no", + "ProtectSystem": "no", + "RefuseManualStart": "no", + "RefuseManualStop": "no", + "RemainAfterExit": "no", + "RemoveIPC": "no", + "Requires": "system-vncserver.slice sysinit.target", + "Restart": "always", + "RestartUSec": "15s", + "RestrictNamespaces": "no", + "RestrictRealtime": "no", + "RestrictSUIDSGID": "no", + "Result": "success", + "RootDirectoryStartOnly": "no", + "RuntimeDirectoryMode": "0755", + "RuntimeDirectoryPreserve": "no", + "RuntimeMaxUSec": "infinity", + "SameProcessGroup": "no", + "SecureBits": "0", + "SendSIGHUP": "no", + "SendSIGKILL": "yes", + "Slice": "system-vncserver.slice", + "StandardError": "inherit", + "StandardInput": "null", + "StandardInputData": "", + "StandardOutput": "journal", + "StartLimitAction": "none", + "StartLimitBurst": "5", + "StartLimitIntervalUSec": "10s", + "StartupBlockIOWeight": "[not set]", + "StartupCPUShares": "[not set]", + "StartupCPUWeight": "[not set]", + "StartupIOWeight": "[not set]", + "StateChangeTimestampMonotonic": "0", + "StateDirectoryMode": "0755", + "StatusErrno": "0", + "StopWhenUnneeded": "no", + "SubState": "dead", + "SuccessAction": "none", + "SyslogFacility": "3", + "SyslogLevel": "6", + "SyslogLevelPrefix": "yes", + "SyslogPriority": "30", + "SystemCallErrorNumber": "0", + "TTYReset": "no", + "TTYVHangup": "no", + "TTYVTDisallocate": "no", + "TasksAccounting": "yes", + "TasksCurrent": "[not set]", + "TasksMax": "203813", + "TimeoutStartUSec": "1min 30s", + "TimeoutStopUSec": "1min 30s", + "TimerSlackNSec": "50000", + "Transient": "no", + "Type": "forking", + "UID": "[not set]", + "UMask": "0022", + "UnitFilePreset": "disabled", + "UnitFileState": "disabled", + "UtmpMode": "init", + "WatchdogTimestampMonotonic": "0", + "WatchdogUSec": "0" + } +} + +TASK [vnc : Install TightVNC] ************************************************** +skipping: [localhost] => { + "changed": false, + "false_condition": "vnc_flavor | lower == 'tightvnc'", + "skip_reason": "Conditional result was False" +} + +TASK [vnc : Set current role for lock manager completion] ********************** +ok: [localhost] => { + "ansible_facts": { + "current_role": "vnc", + "role_completed": true + }, + "changed": false +} + +TASK [lock_manager : Debug create lock operation] ****************************** +ok: [localhost] => {} + +MSG: + +Creating/updating lock file for role: vnc + +TASK [lock_manager : Check if lock file exists] ******************************** +ok: [localhost] => { + "changed": false, + "stat": { + "atime": 1753371018.3885562, + "attr_flags": "", + "attributes": [], + "block_size": 4096, + "blocks": 8, + "charset": "us-ascii", + "checksum": "ca35350cf060981f0667398fcb15c8e12423a97e", + "ctime": 1753371018.1625395, + "dev": 2050, + "device_type": 0, + "executable": false, + "exists": true, + "gid": 0, + "gr_name": "root", + "inode": 117754419, + "isblk": false, + "ischr": false, + "isdir": false, + "isfifo": false, + "isgid": false, + "islnk": false, + "isreg": true, + "issock": false, + "isuid": false, + "mimetype": "text/plain", + "mode": "0644", + "mtime": 1753371018.0155284, + "nlink": 1, + "path": "/opt/vdi-setup/.vdi-lock.yaml", + "pw_name": "root", + "readable": true, + "rgrp": true, + "roth": true, + "rusr": true, + "size": 1024, + "uid": 0, + "version": "3086664638", + "wgrp": false, + "woth": false, + "writeable": true, + "wusr": true, + "xgrp": false, + "xoth": false, + "xusr": false + } +} + +TASK [lock_manager : Debug lock file existence] ******************************** +ok: [localhost] => {} + +MSG: + +Lock file exists: True + +TASK [lock_manager : Create initial lock file structure] *********************** +skipping: [localhost] => { + "changed": false, + "false_condition": "not lock_file_stat.stat.exists", + "skip_reason": "Conditional result was False" +} + +TASK [lock_manager : Debug lock file creation] ********************************* +skipping: [localhost] => { + "false_condition": "debug | default(false) and not lock_file_stat.stat.exists" +} + +TASK [lock_manager : Load existing lock file data] ***************************** +ok: [localhost] => { + "ansible_facts": { + "vdi_setup_status": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:16Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:16Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "ansible_included_var_files": [ + "/opt/vdi-setup/.vdi-lock.yaml" + ], + "changed": false +} + +TASK [lock_manager : Get current lock data] ************************************ +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:16Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:16Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Update deployment hash] *********************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:16Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:16Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Get current timestamp] ************************************ +changed: [localhost] => { + "changed": true, + "cmd": [ + "date", + "-u", + "+%Y-%m-%dT%H:%M:%SZ" + ], + "delta": "0:00:00.020571", + "end": "2025-07-24 15:30:30.669137", + "rc": 0, + "start": "2025-07-24 15:30:30.648566" +} + +STDOUT: + +2025-07-24T15:30:30Z + +TASK [lock_manager : Update last updated timestamp] **************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:30Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:16Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Get current user secrets status] ************************** +ok: [localhost] => { + "ansible_facts": { + "current_user_secrets": { + "last_secret_check": "2025-07-24T15:30:16Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Create user secrets update] ******************************* +ok: [localhost] => { + "ansible_facts": { + "user_secrets_update": { + "last_secret_check": "2025-07-24T15:30:30Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" + } + }, + "changed": false +} + +TASK [lock_manager : Update user secrets hash] ********************************* +ok: [localhost] => { + "ansible_facts": { + "updated_user_secrets": { + "last_secret_check": "2025-07-24T15:30:30Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Apply user secrets update] ******************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:30Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:30Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Create role status entry] ********************************* +ok: [localhost] => { + "ansible_facts": { + "role_status_entry": { + "completed": true, + "completed_at": "2025-07-24T15:30:30Z" + } + }, + "changed": false +} + +TASK [lock_manager : Debug role status entry] ********************************** +ok: [localhost] => {} + +MSG: + +Role status entry for vnc: + - completed: True + - completed_at: 2025-07-24T15:30:30Z + + +TASK [lock_manager : Debug current lock data before role update] *************** +ok: [localhost] => {} + +MSG: + +Current lock data before updating vnc: + - current_lock_data: completed_roles: + base_os: + completed: true + completed_at: '2025-07-24T15:29:50Z' + lock_manager: + completed: true + completed_at: '2025-07-24T15:27:44Z' + secret_manager: + completed: true + completed_at: '2025-07-24T15:30:03Z' + user_provision: + completed: true + completed_at: '2025-07-24T15:30:16Z' + vdi_tool: + completed: false + vnc: + completed: false +created_at: '2025-07-24T15:27:41Z' +deployment_hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab +deployment_name: vdi-test-scott +force_rerun: false +last_updated: '2025-07-24T15:30:30Z' +lock_version: '1.0' +setup_status: configuring +user_management: + current_users: + - alice + - bob + previous_users: + - alice + - bob + removed_users: [] +user_secrets_status: + last_secret_check: '2025-07-24T15:30:30Z' + user_secrets_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 + users_updated: + - alice + - bob + + - current_completed_roles: base_os: + completed: true + completed_at: '2025-07-24T15:29:50Z' +lock_manager: + completed: true + completed_at: '2025-07-24T15:27:44Z' +secret_manager: + completed: true + completed_at: '2025-07-24T15:30:03Z' +user_provision: + completed: true + completed_at: '2025-07-24T15:30:16Z' +vdi_tool: + completed: false +vnc: + completed: false + + - role_completed: True + - current_role: vnc + + +TASK [lock_manager : Get current completed roles] ****************************** +ok: [localhost] => { + "ansible_facts": { + "current_completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": false + } + } + }, + "changed": false +} + +TASK [lock_manager : Update completed roles] *********************************** +ok: [localhost] => { + "ansible_facts": { + "updated_completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": true, + "completed_at": "2025-07-24T15:30:30Z" + } + } + }, + "changed": false +} + +TASK [lock_manager : Update lock data with completed roles] ******************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": true, + "completed_at": "2025-07-24T15:30:30Z" + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:30Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:30Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Debug updated lock data after role update] **************** +ok: [localhost] => {} + +MSG: + +Updated lock data after updating vnc: + - current_role: vnc + - role_completed: True + - completed_roles: {'base_os': {'completed': True, 'completed_at': '2025-07-24T15:29:50Z'}, 'lock_manager': {'completed': True, 'completed_at': '2025-07-24T15:27:44Z'}, 'secret_manager': {'completed': True, 'completed_at': '2025-07-24T15:30:03Z'}, 'user_provision': {'completed': True, 'completed_at': '2025-07-24T15:30:16Z'}, 'vdi_tool': {'completed': False}, 'vnc': {'completed': True, 'completed_at': '2025-07-24T15:30:30Z'}} + + +TASK [lock_manager : Get current users updated list] *************************** +ok: [localhost] => { + "ansible_facts": { + "current_users_updated": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Get new usernames list] *********************************** +ok: [localhost] => { + "ansible_facts": { + "new_usernames": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Combine users updated lists] ****************************** +ok: [localhost] => { + "ansible_facts": { + "combined_users_updated": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Debug user deduplication] ********************************* +ok: [localhost] => {} + +MSG: + +User deduplication for vnc: + - current_users_updated: ['alice', 'bob'] + - new_usernames: ['alice', 'bob'] + - combined_users_updated: ['alice', 'bob'] + + +TASK [lock_manager : Get current user secrets for list update] ***************** +ok: [localhost] => { + "ansible_facts": { + "current_user_secrets_for_list": { + "last_secret_check": "2025-07-24T15:30:30Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Update users updated list] ******************************** +ok: [localhost] => { + "ansible_facts": { + "updated_user_secrets_with_list": { + "last_secret_check": "2025-07-24T15:30:30Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Apply users updated list] ********************************* +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": true, + "completed_at": "2025-07-24T15:30:30Z" + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:30Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:30Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Get current user management data] ************************* +ok: [localhost] => { + "ansible_facts": { + "current_user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + } + }, + "changed": false +} + +TASK [lock_manager : Get current usernames list] ******************************* +ok: [localhost] => { + "ansible_facts": { + "current_usernames": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Update user management data] ****************************** +ok: [localhost] => { + "ansible_facts": { + "updated_user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + } + }, + "changed": false +} + +TASK [lock_manager : Apply user management update] ***************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": true, + "completed_at": "2025-07-24T15:30:30Z" + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:30Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:30Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Debug user management] ************************************ +ok: [localhost] => {} + +MSG: + +User management for vnc: + - current_users: ['alice', 'bob'] + - previous_users: ['alice', 'bob'] + - removed_users: [] + + +TASK [lock_manager : Check if all roles are completed] ************************* +ok: [localhost] => { + "ansible_facts": { + "all_roles_completed": false + }, + "changed": false +} + +TASK [lock_manager : Update setup status based on completion] ****************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": true, + "completed_at": "2025-07-24T15:30:30Z" + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:30Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:30Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Debug setup status update] ******************************** +ok: [localhost] => {} + +MSG: + +Setup status update: + - all_roles_completed: False + - setup_status: configuring + - completed_roles_count: 5 + - total_roles_count: 6 + + +TASK [lock_manager : Write updated lock file] ********************************** +changed: [localhost] => { + "changed": true, + "checksum": "542bc16243f6f0f5bd20bfbc7c7d0ae70dfc3965", + "dest": "/opt/vdi-setup/.vdi-lock.yaml", + "gid": 0, + "group": "root", + "md5sum": "51acde48f43e59dc50980ecbe5a85b15", + "mode": "0644", + "owner": "root", + "size": 1066, + "src": "/root/.ansible/tmp/ansible-tmp-1753371032.3061836-27078-30129676363744/source", + "state": "file", + "uid": 0 +} + +TASK [lock_manager : Debug VM metadata update] ********************************* +ok: [localhost] => {} + +MSG: + +VM metadata update info: + - hostname: vdi-test-scott-rocky-0 + - zone: us-central1-a + - lock_file_changed: True + + +TASK [lock_manager : Get lock file content as base64] ************************** +changed: [localhost] => { + "changed": true, + "cmd": "cat /opt/vdi-setup/.vdi-lock.yaml | base64 -w 0", + "delta": "0:00:00.003949", + "end": "2025-07-24 15:30:32.886270", + "rc": 0, + "start": "2025-07-24 15:30:32.882321" +} + +STDOUT: + +dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiB0cnVlCiAgICAgIGNvbXBsZXRlZF9hdDogJzIwMjUtMDctMjRUMTU6MzA6MTZaJwogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogZmFsc2UKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiB0cnVlCiAgICAgIGNvbXBsZXRlZF9hdDogJzIwMjUtMDctMjRUMTU6MzA6MzBaJwogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNTozMDozMFonCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogY29uZmlndXJpbmcKICB1c2VyX21hbmFnZW1lbnQ6CiAgICBjdXJyZW50X3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHByZXZpb3VzX3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHJlbW92ZWRfdXNlcnM6IFtdCiAgdXNlcl9zZWNyZXRzX3N0YXR1czoKICAgIGxhc3Rfc2VjcmV0X2NoZWNrOiAnMjAyNS0wNy0yNFQxNTozMDozMFonCiAgICB1c2VyX3NlY3JldHNfaGFzaDogNmE2YTAyZDNkMDk5MzNjYTI4MmFlMmIwMjMzNjM0M2M5ZDczNzAxZTQ1YmQ4MTQ3NWZiZDkyZGNmOWQyZjNiNgogICAgdXNlcnNfdXBkYXRlZDoKICAgIC0gYWxpY2UKICAgIC0gYm9iCg== + +TASK [lock_manager : Add VDI lock file content to VM metadata] ***************** +changed: [localhost] => { + "changed": true, + "cmd": [ + "gcloud", + "compute", + "instances", + "add-metadata", + "vdi-test-scott-rocky-0", + "--metadata", + "vdi-lock-content=dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiB0cnVlCiAgICAgIGNvbXBsZXRlZF9hdDogJzIwMjUtMDctMjRUMTU6MzA6MTZaJwogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogZmFsc2UKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiB0cnVlCiAgICAgIGNvbXBsZXRlZF9hdDogJzIwMjUtMDctMjRUMTU6MzA6MzBaJwogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNTozMDozMFonCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogY29uZmlndXJpbmcKICB1c2VyX21hbmFnZW1lbnQ6CiAgICBjdXJyZW50X3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHByZXZpb3VzX3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHJlbW92ZWRfdXNlcnM6IFtdCiAgdXNlcl9zZWNyZXRzX3N0YXR1czoKICAgIGxhc3Rfc2VjcmV0X2NoZWNrOiAnMjAyNS0wNy0yNFQxNTozMDozMFonCiAgICB1c2VyX3NlY3JldHNfaGFzaDogNmE2YTAyZDNkMDk5MzNjYTI4MmFlMmIwMjMzNjM0M2M5ZDczNzAxZTQ1YmQ4MTQ3NWZiZDkyZGNmOWQyZjNiNgogICAgdXNlcnNfdXBkYXRlZDoKICAgIC0gYWxpY2UKICAgIC0gYm9iCg==", + "--zone=us-central1-a" + ], + "delta": "0:00:05.181207", + "end": "2025-07-24 15:30:38.265496", + "failed_when_result": false, + "rc": 0, + "start": "2025-07-24 15:30:33.084289" +} + +STDERR: + +Updated [https://www.googleapis.com/compute/v1/projects/hpc-discovery-external/zones/us-central1-a/instances/vdi-test-scott-rocky-0]. + +TASK [lock_manager : Debug lock file write] ************************************ +ok: [localhost] => {} + +MSG: + +Lock file written: True + +TASK [lock_manager : Verify lock file integrity] ******************************* +ok: [localhost] => { + "changed": false, + "cmd": [ + "python3", + "-c", + "import yaml; yaml.safe_load(open('/opt/vdi-setup/.vdi-lock.yaml'))" + ], + "delta": "0:00:00.040227", + "end": "2025-07-24 15:30:38.534489", + "rc": 0, + "start": "2025-07-24 15:30:38.494262" +} + +TASK [lock_manager : Debug lock file validation] ******************************* +ok: [localhost] => {} + +MSG: + +Lock file validation: True + +TASK [lock_manager : Check if VDI monitor service exists] ********************** +ok: [localhost] => { + "changed": false, + "stat": { + "atime": 1753370874.1548011, + "attr_flags": "", + "attributes": [], + "block_size": 4096, + "blocks": 8, + "charset": "us-ascii", + "checksum": "f6de22707f8d9248014f679a2f1930451e2bf635", + "ctime": 1753370872.6256874, + "dev": 2050, + "device_type": 0, + "executable": false, + "exists": true, + "gid": 0, + "gr_name": "root", + "inode": 285214140, + "isblk": false, + "ischr": false, + "isdir": false, + "isfifo": false, + "isgid": false, + "islnk": false, + "isreg": true, + "issock": false, + "isuid": false, + "mimetype": "text/plain", + "mode": "0644", + "mtime": 1753370872.4766762, + "nlink": 1, + "path": "/etc/systemd/system/vdi-monitor.service", + "pw_name": "root", + "readable": true, + "rgrp": true, + "roth": true, + "rusr": true, + "size": 283, + "uid": 0, + "version": "1338188335", + "wgrp": false, + "woth": false, + "writeable": true, + "wusr": true, + "xgrp": false, + "xoth": false, + "xusr": false + } +} + +TASK [lock_manager : Debug VDI monitor setup conditions] *********************** +ok: [localhost] => {} + +MSG: + +VDI monitor setup conditions: + - current_role: vnc + - vdi_monitor_service_exists: True + - lock_file_creation.changed: False + - will_setup_monitor: False + + +TASK [lock_manager : Setup VDI monitoring service (first time only)] *********** +skipping: [localhost] => { + "changed": false, + "false_condition": "not vdi_monitor_service_stat.stat.exists", + "skip_reason": "Conditional result was False" +} + +TASK [lock_manager : Fail if lock file is invalid] ***************************** +skipping: [localhost] => { + "changed": false, + "false_condition": "lock_file_validation.rc != 0", + "skip_reason": "Conditional result was False" +} + +TASK [vdi_tool : Set current role for lock manager check] ********************** +ok: [localhost] => { + "ansible_facts": { + "current_role": "vdi_tool" + }, + "changed": false +} + +TASK [lock_manager : Check if lock file exists] ******************************** +ok: [localhost] => { + "changed": false, + "stat": { + "atime": 1753371032.8846393, + "attr_flags": "", + "attributes": [], + "block_size": 4096, + "blocks": 8, + "charset": "us-ascii", + "checksum": "542bc16243f6f0f5bd20bfbc7c7d0ae70dfc3965", + "ctime": 1753371032.652622, + "dev": 2050, + "device_type": 0, + "executable": false, + "exists": true, + "gid": 0, + "gr_name": "root", + "inode": 419489989, + "isblk": false, + "ischr": false, + "isdir": false, + "isfifo": false, + "isgid": false, + "islnk": false, + "isreg": true, + "issock": false, + "isuid": false, + "mimetype": "text/plain", + "mode": "0644", + "mtime": 1753371032.49061, + "nlink": 1, + "path": "/opt/vdi-setup/.vdi-lock.yaml", + "pw_name": "root", + "readable": true, + "rgrp": true, + "roth": true, + "rusr": true, + "size": 1066, + "uid": 0, + "version": "68461029", + "wgrp": false, + "woth": false, + "writeable": true, + "wusr": true, + "xgrp": false, + "xoth": false, + "xusr": false + } +} + +TASK [lock_manager : Debug lock file existence] ******************************** +ok: [localhost] => {} + +MSG: + +Lock file exists: True + +TASK [lock_manager : Load existing lock file] ********************************** +ok: [localhost] => { + "ansible_facts": { + "vdi_setup_status": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": true, + "completed_at": "2025-07-24T15:30:30Z" + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:30Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:30Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "ansible_included_var_files": [ + "/opt/vdi-setup/.vdi-lock.yaml" + ], + "changed": false +} + +TASK [lock_manager : Debug lock file load result] ****************************** +ok: [localhost] => {} + +MSG: + +Lock file loaded successfully: True + +TASK [lock_manager : Calculate current deployment hash] ************************ +ok: [localhost] => { + "ansible_facts": { + "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab" + }, + "changed": false +} + +TASK [lock_manager : Debug deployment hash calculation] ************************ +ok: [localhost] => {} + +MSG: + +Current deployment hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab +Deployment components: + - deployment_name: vdi-test-scott + - project_id: hpc-discovery-external + - vdi_tool: guacamole + - vnc_flavor: tigervnc + - user_provision: local_users + - vdi_webapp_port: 8081 + - vdi_resolution: 1920x1080 + - vdi_users count: 2 + + +TASK [lock_manager : Calculate current user secrets hash] ********************** +ok: [localhost] => { + "ansible_facts": { + "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" + }, + "changed": false +} + +TASK [lock_manager : Debug user secrets hash calculation] ********************** +ok: [localhost] => {} + +MSG: + +Current user secrets hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 +User configuration components: + - vdi_users (blueprint): [{"username": "alice", "port": 5901}, {"username": "bob", "port": 5902, "secret_name": "a-password-for-bob"}] + + +TASK [lock_manager : Set default values for missing lock file] ***************** +skipping: [localhost] => { + "changed": false, + "false_condition": "not lock_file_stat.stat.exists", + "skip_reason": "Conditional result was False" +} + +TASK [lock_manager : Debug lock file status] *********************************** +ok: [localhost] => {} + +MSG: + +Lock file status: + - exists: True + - deployment_hash: none + - user_secrets_hash: none + - force_rerun: False + + +TASK [lock_manager : Check if lock file exists] ******************************** +ok: [localhost] => { + "ansible_facts": { + "lock_file_exists": true + }, + "changed": false +} + +TASK [lock_manager : Check if current role is completed] *********************** +ok: [localhost] => { + "ansible_facts": { + "current_role_completed": false + }, + "changed": false +} + +TASK [lock_manager : Check if deployment hash matches] ************************* +ok: [localhost] => { + "ansible_facts": { + "deployment_hash_matches": false + }, + "changed": false +} + +TASK [lock_manager : Check if force rerun is enabled] ************************** +ok: [localhost] => { + "ansible_facts": { + "force_rerun_enabled": false + }, + "changed": false +} + +TASK [lock_manager : Determine if role should run] ***************************** +ok: [localhost] => { + "ansible_facts": { + "role_should_run": true + }, + "changed": false +} + +TASK [lock_manager : Debug role should run calculation] ************************ +ok: [localhost] => {} + +MSG: + +Role should run calculation for vdi_tool: + - not lock_file_exists: False + - not current_role_completed: True + - not deployment_hash_matches: True + - force_rerun_enabled: False + - role_should_run: True + + +TASK [lock_manager : Debug role execution decision] **************************** +ok: [localhost] => {} + +MSG: + +Role execution decision for vdi_tool: + - current_role: vdi_tool + - ansible_role_name: lock_manager + - role_should_run: True + - lock_file_exists: True + - current_role_completed: False + - deployment_hash_matches: False + - force_rerun_enabled: False + + +TASK [lock_manager : Check if user configuration has changed] ****************** +ok: [localhost] => { + "ansible_facts": { + "user_config_changed": true + }, + "changed": false +} + +TASK [lock_manager : Set user secrets changed flag (will be updated by secret_manager role)] *** +ok: [localhost] => { + "ansible_facts": { + "user_secrets_changed": true + }, + "changed": false +} + +TASK [lock_manager : Debug user secrets change detection] ********************** +ok: [localhost] => {} + +MSG: + +User configuration change detection: + - user_config_changed: True + - user_secrets_changed: True + - stored_hash: none + - current_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 + + +TASK [lock_manager : Set lock check result] ************************************ +ok: [localhost] => { + "ansible_facts": { + "lock_check_result": { + "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "existing_deployment_hash": "none", + "existing_user_secrets_hash": "none", + "force_rerun": false, + "role_should_run": true, + "user_secrets_changed": true + } + }, + "changed": false +} + +TASK [lock_manager : Debug final lock check result] **************************** +ok: [localhost] => {} + +MSG: + +Final lock check result: + - role_should_run: True + - deployment_hash_changed: True + - user_secrets_changed: True + - force_rerun: False + + +TASK [vdi_tool : Set role_should_run from lock check result] ******************* +ok: [localhost] => { + "ansible_facts": { + "role_should_run": true + }, + "changed": false +} + +TASK [vdi_tool : Skip vdi_tool tasks if role should not run] ******************* +skipping: [localhost] => { + "false_condition": "not role_should_run" +} + +TASK [vdi_tool : Install Guacamole] ******************************************** +included: /tmp/vdi/roles/vdi_tool/tasks/guacamole.yaml for localhost + +TASK [vdi_tool : Ensure PostgreSQL data & init directories exist] ************** +changed: [localhost] => (item=/opt/guacamole-db/data) => { + "ansible_loop_var": "item", + "changed": true, + "gid": 0, + "group": "root", + "item": "/opt/guacamole-db/data", + "mode": "0755", + "owner": "root", + "path": "/opt/guacamole-db/data", + "size": 6, + "state": "directory", + "uid": 0 +} +changed: [localhost] => (item=/opt/guacamole-db/initdb) => { + "ansible_loop_var": "item", + "changed": true, + "gid": 0, + "group": "root", + "item": "/opt/guacamole-db/initdb", + "mode": "0755", + "owner": "root", + "path": "/opt/guacamole-db/initdb", + "size": 6, + "state": "directory", + "uid": 0 +} + +TASK [vdi_tool : Check Docker service status] ********************************** +ok: [localhost] => { + "ansible_facts": { + "services": { + "NetworkManager-dispatcher.service": { + "name": "NetworkManager-dispatcher.service", + "source": "systemd", + "state": "inactive", + "status": "enabled" + }, + "NetworkManager-wait-online.service": { + "name": "NetworkManager-wait-online.service", + "source": "systemd", + "state": "stopped", + "status": "enabled" + }, + "NetworkManager.service": { + "name": "NetworkManager.service", + "source": "systemd", + "state": "running", + "status": "enabled" + }, + "accounts-daemon.service": { + "name": "accounts-daemon.service", + "source": "systemd", + "state": "running", + "status": "enabled" + }, + "acpid.service": { + "name": "acpid.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "apt-daily.service": { + "name": "apt-daily.service", + "source": "systemd", + "state": "stopped", + "status": "not-found" + }, + "arp-ethers.service": { + "name": "arp-ethers.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "atd.service": { + "name": "atd.service", + "source": "systemd", + "state": "running", + "status": "enabled" + }, + "auditd.service": { + "name": "auditd.service", + "source": "systemd", + "state": "running", + "status": "enabled" + }, + "auth-rpcgss-module.service": { + "name": "auth-rpcgss-module.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "auto-cpufreq.service": { + "name": "auto-cpufreq.service", + "source": "systemd", + "state": "stopped", + "status": "not-found" + }, + "autovt@.service": { + "name": "autovt@.service", + "source": "systemd", + "state": "unknown", + "status": "enabled" + }, + "bluetooth.service": { + "name": "bluetooth.service", + "source": "systemd", + "state": "inactive", + "status": "enabled" + }, + "bolt.service": { + "name": "bolt.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "btattach-bcm@.service": { + "name": "btattach-bcm@.service", + "source": "systemd", + "state": "unknown", + "status": "static" + }, + "canberra-system-bootup.service": { + "name": "canberra-system-bootup.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "canberra-system-shutdown-reboot.service": { + "name": "canberra-system-shutdown-reboot.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "canberra-system-shutdown.service": { + "name": "canberra-system-shutdown.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "chrony-dnssrv@.service": { + "name": "chrony-dnssrv@.service", + "source": "systemd", + "state": "unknown", + "status": "static" + }, + "chrony-wait.service": { + "name": "chrony-wait.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "chronyd.service": { + "name": "chronyd.service", + "source": "systemd", + "state": "running", + "status": "enabled" + }, + "colord.service": { + "name": "colord.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "console-getty.service": { + "name": "console-getty.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "container-getty@.service": { + "name": "container-getty@.service", + "source": "systemd", + "state": "unknown", + "status": "static" + }, + "containerd.service": { + "name": "containerd.service", + "source": "systemd", + "state": "running", + "status": "disabled" + }, + "cpupower.service": { + "name": "cpupower.service", + "source": "systemd", + "state": "stopped", + "status": "disabled" + }, + "crond.service": { + "name": "crond.service", + "source": "systemd", + "state": "running", + "status": "enabled" + }, + "daos_agent.service": { + "name": "daos_agent.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "dbus-org.bluez.service": { + "name": "dbus-org.bluez.service", + "source": "systemd", + "state": "inactive", + "status": "enabled" + }, + "dbus-org.freedesktop.hostname1.service": { + "name": "dbus-org.freedesktop.hostname1.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "dbus-org.freedesktop.locale1.service": { + "name": "dbus-org.freedesktop.locale1.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "dbus-org.freedesktop.login1.service": { + "name": "dbus-org.freedesktop.login1.service", + "source": "systemd", + "state": "active", + "status": "static" + }, + "dbus-org.freedesktop.nm-dispatcher.service": { + "name": "dbus-org.freedesktop.nm-dispatcher.service", + "source": "systemd", + "state": "inactive", + "status": "enabled" + }, + "dbus-org.freedesktop.portable1.service": { + "name": "dbus-org.freedesktop.portable1.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "dbus-org.freedesktop.timedate1.service": { + "name": "dbus-org.freedesktop.timedate1.service", + "source": "systemd", + "state": "inactive", + "status": "enabled" + }, + "dbus.service": { + "name": "dbus.service", + "source": "systemd", + "state": "running", + "status": "static" + }, + "debug-shell.service": { + "name": "debug-shell.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "display-manager.service": { + "name": "display-manager.service", + "source": "systemd", + "state": "inactive", + "status": "enabled" + }, + "dnf-automatic-download.service": { + "name": "dnf-automatic-download.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "dnf-automatic-install.service": { + "name": "dnf-automatic-install.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "dnf-automatic-notifyonly.service": { + "name": "dnf-automatic-notifyonly.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "dnf-automatic.service": { + "name": "dnf-automatic.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "dnf-makecache.service": { + "name": "dnf-makecache.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "dnf-system-upgrade-cleanup.service": { + "name": "dnf-system-upgrade-cleanup.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "dnf-system-upgrade.service": { + "name": "dnf-system-upgrade.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "docker.service": { + "name": "docker.service", + "source": "systemd", + "state": "running", + "status": "enabled" + }, + "dracut-cmdline.service": { + "name": "dracut-cmdline.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "dracut-initqueue.service": { + "name": "dracut-initqueue.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "dracut-mount.service": { + "name": "dracut-mount.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "dracut-pre-mount.service": { + "name": "dracut-pre-mount.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "dracut-pre-pivot.service": { + "name": "dracut-pre-pivot.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "dracut-pre-trigger.service": { + "name": "dracut-pre-trigger.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "dracut-pre-udev.service": { + "name": "dracut-pre-udev.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "dracut-shutdown-onfailure.service": { + "name": "dracut-shutdown-onfailure.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "dracut-shutdown.service": { + "name": "dracut-shutdown.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "ebtables.service": { + "name": "ebtables.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "emergency.service": { + "name": "emergency.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "firewalld.service": { + "name": "firewalld.service", + "source": "systemd", + "state": "stopped", + "status": "masked" + }, + "flatpak-system-helper.service": { + "name": "flatpak-system-helper.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "fstrim.service": { + "name": "fstrim.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "fwupd-offline-update.service": { + "name": "fwupd-offline-update.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "fwupd-refresh.service": { + "name": "fwupd-refresh.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "fwupd.service": { + "name": "fwupd.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "gce-workload-cert-refresh.service": { + "name": "gce-workload-cert-refresh.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "gdm.service": { + "name": "gdm.service", + "source": "systemd", + "state": "stopped", + "status": "enabled" + }, + "geoclue.service": { + "name": "geoclue.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "getty@.service": { + "name": "getty@.service", + "source": "systemd", + "state": "unknown", + "status": "enabled" + }, + "getty@tty1.service": { + "name": "getty@tty1.service", + "source": "systemd", + "state": "running", + "status": "active" + }, + "google-disk-expand.service": { + "name": "google-disk-expand.service", + "source": "systemd", + "state": "stopped", + "status": "enabled" + }, + "google-guest-agent-manager.service": { + "name": "google-guest-agent-manager.service", + "source": "systemd", + "state": "running", + "status": "enabled" + }, + "google-guest-agent.service": { + "name": "google-guest-agent.service", + "source": "systemd", + "state": "running", + "status": "enabled" + }, + "google-guest-compat-manager.service": { + "name": "google-guest-compat-manager.service", + "source": "systemd", + "state": "running", + "status": "enabled" + }, + "google-hpc-firstrun.service": { + "name": "google-hpc-firstrun.service", + "source": "systemd", + "state": "inactive", + "status": "enabled" + }, + "google-hpc-multiqueue.service": { + "name": "google-hpc-multiqueue.service", + "source": "systemd", + "state": "stopped", + "status": "enabled" + }, + "google-osconfig-agent.service": { + "name": "google-osconfig-agent.service", + "source": "systemd", + "state": "running", + "status": "enabled" + }, + "google-oslogin-cache.service": { + "name": "google-oslogin-cache.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "google-shutdown-scripts.service": { + "name": "google-shutdown-scripts.service", + "source": "systemd", + "state": "stopped", + "status": "enabled" + }, + "google-startup-scripts.service": { + "name": "google-startup-scripts.service", + "source": "systemd", + "state": "stopped", + "status": "enabled" + }, + "grub-boot-indeterminate.service": { + "name": "grub-boot-indeterminate.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "gssproxy.service": { + "name": "gssproxy.service", + "source": "systemd", + "state": "running", + "status": "disabled" + }, + "halt-local.service": { + "name": "halt-local.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "hwloc-dump-hwdata.service": { + "name": "hwloc-dump-hwdata.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "iio-sensor-proxy.service": { + "name": "iio-sensor-proxy.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "import-state.service": { + "name": "import-state.service", + "source": "systemd", + "state": "stopped", + "status": "enabled" + }, + "initrd-cleanup.service": { + "name": "initrd-cleanup.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "initrd-parse-etc.service": { + "name": "initrd-parse-etc.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "initrd-switch-root.service": { + "name": "initrd-switch-root.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "initrd-udevadm-cleanup-db.service": { + "name": "initrd-udevadm-cleanup-db.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "iprdump.service": { + "name": "iprdump.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "iprinit.service": { + "name": "iprinit.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "iprupdate.service": { + "name": "iprupdate.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "kdump.service": { + "name": "kdump.service", + "source": "systemd", + "state": "stopped", + "status": "enabled" + }, + "kmod-static-nodes.service": { + "name": "kmod-static-nodes.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "kvm_stat.service": { + "name": "kvm_stat.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "ldconfig.service": { + "name": "ldconfig.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "loadmodules.service": { + "name": "loadmodules.service", + "source": "systemd", + "state": "stopped", + "status": "enabled" + }, + "man-db-cache-update.service": { + "name": "man-db-cache-update.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "man-db-restart-cache-update.service": { + "name": "man-db-restart-cache-update.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "mdadm-grow-continue@.service": { + "name": "mdadm-grow-continue@.service", + "source": "systemd", + "state": "unknown", + "status": "static" + }, + "mdadm-last-resort@.service": { + "name": "mdadm-last-resort@.service", + "source": "systemd", + "state": "unknown", + "status": "static" + }, + "mdcheck_continue.service": { + "name": "mdcheck_continue.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "mdcheck_start.service": { + "name": "mdcheck_start.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "mdmon@.service": { + "name": "mdmon@.service", + "source": "systemd", + "state": "unknown", + "status": "static" + }, + "mdmonitor-oneshot.service": { + "name": "mdmonitor-oneshot.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "mdmonitor.service": { + "name": "mdmonitor.service", + "source": "systemd", + "state": "stopped", + "status": "enabled" + }, + "messagebus.service": { + "name": "messagebus.service", + "source": "systemd", + "state": "active", + "status": "static" + }, + "modprobe@efi_pstore.service": { + "name": "modprobe@efi_pstore.service", + "source": "systemd", + "state": "stopped", + "status": "not-found" + }, + "mount-localssd-raid.service": { + "name": "mount-localssd-raid.service", + "source": "systemd", + "state": "stopped", + "status": "not-found" + }, + "network.service": { + "name": "network.service", + "source": "systemd", + "state": "stopped", + "status": "not-found" + }, + "networking.service": { + "name": "networking.service", + "source": "systemd", + "state": "stopped", + "status": "not-found" + }, + "nfs-blkmap.service": { + "name": "nfs-blkmap.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "nfs-convert.service": { + "name": "nfs-convert.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "nfs-idmapd.service": { + "name": "nfs-idmapd.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "nfs-mountd.service": { + "name": "nfs-mountd.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "nfs-server.service": { + "name": "nfs-server.service", + "source": "systemd", + "state": "stopped", + "status": "disabled" + }, + "nfs-utils.service": { + "name": "nfs-utils.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "nfsdcld.service": { + "name": "nfsdcld.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "nftables.service": { + "name": "nftables.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "nis-domainname.service": { + "name": "nis-domainname.service", + "source": "systemd", + "state": "stopped", + "status": "enabled" + }, + "ntpd.service": { + "name": "ntpd.service", + "source": "systemd", + "state": "stopped", + "status": "not-found" + }, + "ntpdate.service": { + "name": "ntpdate.service", + "source": "systemd", + "state": "stopped", + "status": "not-found" + }, + "nvmefc-boot-connections.service": { + "name": "nvmefc-boot-connections.service", + "source": "systemd", + "state": "inactive", + "status": "enabled" + }, + "nvmf-autoconnect.service": { + "name": "nvmf-autoconnect.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "nvmf-connect@.service": { + "name": "nvmf-connect@.service", + "source": "systemd", + "state": "unknown", + "status": "static" + }, + "pesign.service": { + "name": "pesign.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "plymouth-halt.service": { + "name": "plymouth-halt.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "plymouth-kexec.service": { + "name": "plymouth-kexec.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "plymouth-poweroff.service": { + "name": "plymouth-poweroff.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "plymouth-quit-wait.service": { + "name": "plymouth-quit-wait.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "plymouth-quit.service": { + "name": "plymouth-quit.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "plymouth-read-write.service": { + "name": "plymouth-read-write.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "plymouth-reboot.service": { + "name": "plymouth-reboot.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "plymouth-start.service": { + "name": "plymouth-start.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "plymouth-switch-root-initramfs.service": { + "name": "plymouth-switch-root-initramfs.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "plymouth-switch-root.service": { + "name": "plymouth-switch-root.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "polkit.service": { + "name": "polkit.service", + "source": "systemd", + "state": "running", + "status": "static" + }, + "power-profiles-daemon.service": { + "name": "power-profiles-daemon.service", + "source": "systemd", + "state": "stopped", + "status": "not-found" + }, + "qemu-guest-agent.service": { + "name": "qemu-guest-agent.service", + "source": "systemd", + "state": "inactive", + "status": "enabled" + }, + "quotaon.service": { + "name": "quotaon.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "rc-local.service": { + "name": "rc-local.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "rdisc.service": { + "name": "rdisc.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "rdma-load-modules@.service": { + "name": "rdma-load-modules@.service", + "source": "systemd", + "state": "unknown", + "status": "static" + }, + "rdma-ndd.service": { + "name": "rdma-ndd.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "rescue.service": { + "name": "rescue.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "rngd.service": { + "name": "rngd.service", + "source": "systemd", + "state": "running", + "status": "enabled" + }, + "rpc-gssd.service": { + "name": "rpc-gssd.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "rpc-statd-notify.service": { + "name": "rpc-statd-notify.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "rpc-statd.service": { + "name": "rpc-statd.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "rpcbind.service": { + "name": "rpcbind.service", + "source": "systemd", + "state": "running", + "status": "enabled" + }, + "rsyslog.service": { + "name": "rsyslog.service", + "source": "systemd", + "state": "running", + "status": "enabled" + }, + "rtkit-daemon.service": { + "name": "rtkit-daemon.service", + "source": "systemd", + "state": "running", + "status": "enabled" + }, + "selinux-autorelabel-mark.service": { + "name": "selinux-autorelabel-mark.service", + "source": "systemd", + "state": "stopped", + "status": "enabled" + }, + "selinux-autorelabel.service": { + "name": "selinux-autorelabel.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "serial-getty@.service": { + "name": "serial-getty@.service", + "source": "systemd", + "state": "unknown", + "status": "indirect" + }, + "serial-getty@ttyS0.service": { + "name": "serial-getty@ttyS0.service", + "source": "systemd", + "state": "running", + "status": "active" + }, + "serial-getty@ttyS1.service": { + "name": "serial-getty@ttyS1.service", + "source": "systemd", + "state": "running", + "status": "active" + }, + "sntp.service": { + "name": "sntp.service", + "source": "systemd", + "state": "stopped", + "status": "not-found" + }, + "sshd-keygen@.service": { + "name": "sshd-keygen@.service", + "source": "systemd", + "state": "unknown", + "status": "disabled" + }, + "sshd-keygen@ecdsa.service": { + "name": "sshd-keygen@ecdsa.service", + "source": "systemd", + "state": "stopped", + "status": "inactive" + }, + "sshd-keygen@ed25519.service": { + "name": "sshd-keygen@ed25519.service", + "source": "systemd", + "state": "stopped", + "status": "inactive" + }, + "sshd-keygen@rsa.service": { + "name": "sshd-keygen@rsa.service", + "source": "systemd", + "state": "stopped", + "status": "inactive" + }, + "sshd.service": { + "name": "sshd.service", + "source": "systemd", + "state": "running", + "status": "enabled" + }, + "sshd@.service": { + "name": "sshd@.service", + "source": "systemd", + "state": "unknown", + "status": "static" + }, + "sssd-autofs.service": { + "name": "sssd-autofs.service", + "source": "systemd", + "state": "inactive", + "status": "indirect" + }, + "sssd-kcm.service": { + "name": "sssd-kcm.service", + "source": "systemd", + "state": "stopped", + "status": "indirect" + }, + "sssd-nss.service": { + "name": "sssd-nss.service", + "source": "systemd", + "state": "inactive", + "status": "indirect" + }, + "sssd-pac.service": { + "name": "sssd-pac.service", + "source": "systemd", + "state": "inactive", + "status": "indirect" + }, + "sssd-pam.service": { + "name": "sssd-pam.service", + "source": "systemd", + "state": "inactive", + "status": "indirect" + }, + "sssd-ssh.service": { + "name": "sssd-ssh.service", + "source": "systemd", + "state": "inactive", + "status": "indirect" + }, + "sssd-sudo.service": { + "name": "sssd-sudo.service", + "source": "systemd", + "state": "inactive", + "status": "indirect" + }, + "sssd.service": { + "name": "sssd.service", + "source": "systemd", + "state": "stopped", + "status": "enabled" + }, + "switcheroo-control.service": { + "name": "switcheroo-control.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "syslog.service": { + "name": "syslog.service", + "source": "systemd", + "state": "stopped", + "status": "not-found" + }, + "system-update-cleanup.service": { + "name": "system-update-cleanup.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "systemd-ask-password-console.service": { + "name": "systemd-ask-password-console.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-ask-password-plymouth.service": { + "name": "systemd-ask-password-plymouth.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-ask-password-wall.service": { + "name": "systemd-ask-password-wall.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-backlight@.service": { + "name": "systemd-backlight@.service", + "source": "systemd", + "state": "unknown", + "status": "static" + }, + "systemd-binfmt.service": { + "name": "systemd-binfmt.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-coredump@.service": { + "name": "systemd-coredump@.service", + "source": "systemd", + "state": "unknown", + "status": "static" + }, + "systemd-exit.service": { + "name": "systemd-exit.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "systemd-firstboot.service": { + "name": "systemd-firstboot.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-fsck-root.service": { + "name": "systemd-fsck-root.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-fsck@.service": { + "name": "systemd-fsck@.service", + "source": "systemd", + "state": "unknown", + "status": "static" + }, + "systemd-fsck@dev-disk-by\\x2duuid-580C\\x2d02F4.service": { + "name": "systemd-fsck@dev-disk-by\\x2duuid-580C\\x2d02F4.service", + "source": "systemd", + "state": "stopped", + "status": "active" + }, + "systemd-halt.service": { + "name": "systemd-halt.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "systemd-hibernate-resume@.service": { + "name": "systemd-hibernate-resume@.service", + "source": "systemd", + "state": "unknown", + "status": "static" + }, + "systemd-hibernate.service": { + "name": "systemd-hibernate.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "systemd-hostnamed.service": { + "name": "systemd-hostnamed.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "systemd-hwdb-update.service": { + "name": "systemd-hwdb-update.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-hybrid-sleep.service": { + "name": "systemd-hybrid-sleep.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "systemd-initctl.service": { + "name": "systemd-initctl.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-journal-catalog-update.service": { + "name": "systemd-journal-catalog-update.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-journal-flush.service": { + "name": "systemd-journal-flush.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-journald.service": { + "name": "systemd-journald.service", + "source": "systemd", + "state": "running", + "status": "static" + }, + "systemd-kexec.service": { + "name": "systemd-kexec.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "systemd-localed.service": { + "name": "systemd-localed.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "systemd-logind.service": { + "name": "systemd-logind.service", + "source": "systemd", + "state": "running", + "status": "static" + }, + "systemd-machine-id-commit.service": { + "name": "systemd-machine-id-commit.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-modules-load.service": { + "name": "systemd-modules-load.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-networkd.service": { + "name": "systemd-networkd.service", + "source": "systemd", + "state": "stopped", + "status": "not-found" + }, + "systemd-portabled.service": { + "name": "systemd-portabled.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "systemd-poweroff.service": { + "name": "systemd-poweroff.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "systemd-pstore.service": { + "name": "systemd-pstore.service", + "source": "systemd", + "state": "stopped", + "status": "enabled" + }, + "systemd-quotacheck.service": { + "name": "systemd-quotacheck.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-random-seed.service": { + "name": "systemd-random-seed.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-reboot.service": { + "name": "systemd-reboot.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "systemd-remount-fs.service": { + "name": "systemd-remount-fs.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-resolved.service": { + "name": "systemd-resolved.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "systemd-rfkill.service": { + "name": "systemd-rfkill.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "systemd-suspend-then-hibernate.service": { + "name": "systemd-suspend-then-hibernate.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "systemd-suspend.service": { + "name": "systemd-suspend.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "systemd-sysctl.service": { + "name": "systemd-sysctl.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-sysusers.service": { + "name": "systemd-sysusers.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-timedated.service": { + "name": "systemd-timedated.service", + "source": "systemd", + "state": "inactive", + "status": "masked" + }, + "systemd-timesyncd.service": { + "name": "systemd-timesyncd.service", + "source": "systemd", + "state": "stopped", + "status": "not-found" + }, + "systemd-tmpfiles-clean.service": { + "name": "systemd-tmpfiles-clean.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-tmpfiles-setup-dev.service": { + "name": "systemd-tmpfiles-setup-dev.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-tmpfiles-setup.service": { + "name": "systemd-tmpfiles-setup.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-udev-settle.service": { + "name": "systemd-udev-settle.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "systemd-udev-trigger.service": { + "name": "systemd-udev-trigger.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-udevd.service": { + "name": "systemd-udevd.service", + "source": "systemd", + "state": "running", + "status": "static" + }, + "systemd-update-done.service": { + "name": "systemd-update-done.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-update-utmp-runlevel.service": { + "name": "systemd-update-utmp-runlevel.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-update-utmp.service": { + "name": "systemd-update-utmp.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-user-sessions.service": { + "name": "systemd-user-sessions.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-vconsole-setup.service": { + "name": "systemd-vconsole-setup.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "systemd-volatile-root.service": { + "name": "systemd-volatile-root.service", + "source": "systemd", + "state": "inactive", + "status": "static" + }, + "tcsd.service": { + "name": "tcsd.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "teamd@.service": { + "name": "teamd@.service", + "source": "systemd", + "state": "unknown", + "status": "static" + }, + "timedatex.service": { + "name": "timedatex.service", + "source": "systemd", + "state": "inactive", + "status": "enabled" + }, + "tlp.service": { + "name": "tlp.service", + "source": "systemd", + "state": "stopped", + "status": "not-found" + }, + "tuned.service": { + "name": "tuned.service", + "source": "systemd", + "state": "running", + "status": "enabled" + }, + "udisks2.service": { + "name": "udisks2.service", + "source": "systemd", + "state": "stopped", + "status": "enabled" + }, + "unbound-anchor.service": { + "name": "unbound-anchor.service", + "source": "systemd", + "state": "stopped", + "status": "static" + }, + "upower.service": { + "name": "upower.service", + "source": "systemd", + "state": "running", + "status": "disabled" + }, + "user-runtime-dir@.service": { + "name": "user-runtime-dir@.service", + "source": "systemd", + "state": "unknown", + "status": "static" + }, + "user-runtime-dir@1000.service": { + "name": "user-runtime-dir@1000.service", + "source": "systemd", + "state": "stopped", + "status": "active" + }, + "user-runtime-dir@1001.service": { + "name": "user-runtime-dir@1001.service", + "source": "systemd", + "state": "stopped", + "status": "active" + }, + "user-runtime-dir@87404603.service": { + "name": "user-runtime-dir@87404603.service", + "source": "systemd", + "state": "stopped", + "status": "active" + }, + "user@.service": { + "name": "user@.service", + "source": "systemd", + "state": "unknown", + "status": "static" + }, + "user@1000.service": { + "name": "user@1000.service", + "source": "systemd", + "state": "running", + "status": "active" + }, + "user@1001.service": { + "name": "user@1001.service", + "source": "systemd", + "state": "running", + "status": "active" + }, + "user@87404603.service": { + "name": "user@87404603.service", + "source": "systemd", + "state": "running", + "status": "active" + }, + "vdi-monitor.service": { + "name": "vdi-monitor.service", + "source": "systemd", + "state": "running", + "status": "enabled" + }, + "vncserver@.service": { + "name": "vncserver@.service", + "source": "systemd", + "state": "unknown", + "status": "indirect" + }, + "vncserver@:1.service": { + "name": "vncserver@:1.service", + "source": "systemd", + "state": "running", + "status": "active" + }, + "vncserver@:2.service": { + "name": "vncserver@:2.service", + "source": "systemd", + "state": "running", + "status": "active" + }, + "wacom-inputattach@.service": { + "name": "wacom-inputattach@.service", + "source": "systemd", + "state": "unknown", + "status": "static" + }, + "wpa_supplicant.service": { + "name": "wpa_supplicant.service", + "source": "systemd", + "state": "inactive", + "status": "disabled" + }, + "xvnc@.service": { + "name": "xvnc@.service", + "source": "systemd", + "state": "unknown", + "status": "static" + }, + "ypbind.service": { + "name": "ypbind.service", + "source": "systemd", + "state": "stopped", + "status": "not-found" + }, + "yppasswdd.service": { + "name": "yppasswdd.service", + "source": "systemd", + "state": "stopped", + "status": "not-found" + }, + "ypserv.service": { + "name": "ypserv.service", + "source": "systemd", + "state": "stopped", + "status": "not-found" + }, + "ypxfrd.service": { + "name": "ypxfrd.service", + "source": "systemd", + "state": "stopped", + "status": "not-found" + } + } + }, + "changed": false +} + +TASK [vdi_tool : Ensure Docker service is running] ***************************** +ok: [localhost] => { + "changed": false, + "enabled": true, + "name": "docker", + "state": "started", + "status": { + "ActiveEnterTimestamp": "Thu 2025-07-24 15:27:29 UTC", + "ActiveEnterTimestampMonotonic": "259862320", + "ActiveExitTimestampMonotonic": "0", + "ActiveState": "active", + "After": "network-online.target containerd.service sysinit.target systemd-journald.socket basic.target docker.socket firewalld.service time-set.target system.slice nss-lookup.target mount-localssd-raid.service", + "AllowIsolate": "no", + "AllowedCPUs": "", + "AllowedMemoryNodes": "", + "AmbientCapabilities": "", + "AssertResult": "yes", + "AssertTimestamp": "Thu 2025-07-24 15:27:26 UTC", + "AssertTimestampMonotonic": "257032900", + "Before": "shutdown.target multi-user.target", + "BlockIOAccounting": "no", + "BlockIOWeight": "[not set]", + "CPUAccounting": "no", + "CPUAffinity": "", + "CPUAffinityFromNUMA": "no", + "CPUQuotaPerSecUSec": "infinity", + "CPUQuotaPeriodUSec": "infinity", + "CPUSchedulingPolicy": "0", + "CPUSchedulingPriority": "0", + "CPUSchedulingResetOnFork": "no", + "CPUShares": "[not set]", + "CPUUsageNSec": "[not set]", + "CPUWeight": "[not set]", + "CacheDirectoryMode": "0755", + "CanFreeze": "yes", + "CanIsolate": "no", + "CanReload": "yes", + "CanStart": "yes", + "CanStop": "yes", + "CapabilityBoundingSet": "cap_chown cap_dac_override cap_dac_read_search cap_fowner cap_fsetid cap_kill cap_setgid cap_setuid cap_setpcap cap_linux_immutable cap_net_bind_service cap_net_broadcast cap_net_admin cap_net_raw cap_ipc_lock cap_ipc_owner cap_sys_module cap_sys_rawio cap_sys_chroot cap_sys_ptrace cap_sys_pacct cap_sys_admin cap_sys_boot cap_sys_nice cap_sys_resource cap_sys_time cap_sys_tty_config cap_mknod cap_lease cap_audit_write cap_audit_control cap_setfcap cap_mac_override cap_mac_admin cap_syslog cap_wake_alarm cap_block_suspend cap_audit_read cap_perfmon cap_bpf", + "CollectMode": "inactive", + "ConditionResult": "yes", + "ConditionTimestamp": "Thu 2025-07-24 15:27:26 UTC", + "ConditionTimestampMonotonic": "257032900", + "ConfigurationDirectoryMode": "0755", + "Conflicts": "shutdown.target", + "ControlGroup": "/system.slice/docker.service", + "ControlPID": "0", + "DefaultDependencies": "yes", + "DefaultMemoryLow": "0", + "DefaultMemoryMin": "0", + "Delegate": "yes", + "DelegateControllers": "cpu cpuacct cpuset io blkio memory devices pids", + "Description": "Docker Application Container Engine", + "DevicePolicy": "auto", + "Documentation": "https://docs.docker.com", + "DropInPaths": "/etc/systemd/system/docker.service.d/data-root.conf", + "DynamicUser": "no", + "EffectiveCPUs": "", + "EffectiveMemoryNodes": "", + "ExecMainCode": "0", + "ExecMainExitTimestampMonotonic": "0", + "ExecMainPID": "5274", + "ExecMainStartTimestamp": "Thu 2025-07-24 15:27:26 UTC", + "ExecMainStartTimestampMonotonic": "257033635", + "ExecMainStatus": "0", + "ExecReload": "{ path=/bin/kill ; argv[]=/bin/kill -s HUP $MAINPID ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", + "ExecStart": "{ path=/usr/bin/dockerd ; argv[]=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", + "FailureAction": "none", + "FileDescriptorStoreMax": "0", + "FragmentPath": "/usr/lib/systemd/system/docker.service", + "FreezerState": "running", + "GID": "[not set]", + "GuessMainPID": "yes", + "IOAccounting": "no", + "IOSchedulingClass": "0", + "IOSchedulingPriority": "0", + "IOWeight": "[not set]", + "IPAccounting": "no", + "IPEgressBytes": "18446744073709551615", + "IPEgressPackets": "18446744073709551615", + "IPIngressBytes": "18446744073709551615", + "IPIngressPackets": "18446744073709551615", + "Id": "docker.service", + "IgnoreOnIsolate": "no", + "IgnoreSIGPIPE": "yes", + "InactiveEnterTimestampMonotonic": "0", + "InactiveExitTimestamp": "Thu 2025-07-24 15:27:26 UTC", + "InactiveExitTimestampMonotonic": "257033661", + "InvocationID": "2c563fb7673f41a09f94812ff7281e0d", + "JobRunningTimeoutUSec": "infinity", + "JobTimeoutAction": "none", + "JobTimeoutUSec": "infinity", + "KeyringMode": "private", + "KillMode": "process", + "KillSignal": "15", + "LimitAS": "infinity", + "LimitASSoft": "infinity", + "LimitCORE": "infinity", + "LimitCORESoft": "infinity", + "LimitCPU": "infinity", + "LimitCPUSoft": "infinity", + "LimitDATA": "infinity", + "LimitDATASoft": "infinity", + "LimitFSIZE": "infinity", + "LimitFSIZESoft": "infinity", + "LimitLOCKS": "infinity", + "LimitLOCKSSoft": "infinity", + "LimitMEMLOCK": "65536", + "LimitMEMLOCKSoft": "65536", + "LimitMSGQUEUE": "819200", + "LimitMSGQUEUESoft": "819200", + "LimitNICE": "0", + "LimitNICESoft": "0", + "LimitNOFILE": "262144", + "LimitNOFILESoft": "1024", + "LimitNPROC": "infinity", + "LimitNPROCSoft": "infinity", + "LimitRSS": "infinity", + "LimitRSSSoft": "infinity", + "LimitRTPRIO": "0", + "LimitRTPRIOSoft": "0", + "LimitRTTIME": "infinity", + "LimitRTTIMESoft": "infinity", + "LimitSIGPENDING": "127383", + "LimitSIGPENDINGSoft": "127383", + "LimitSTACK": "infinity", + "LimitSTACKSoft": "8388608", + "LoadState": "loaded", + "LockPersonality": "no", + "LogLevelMax": "-1", + "LogRateLimitBurst": "0", + "LogRateLimitIntervalUSec": "0", + "LogsDirectoryMode": "0755", + "MainPID": "5274", + "MemoryAccounting": "yes", + "MemoryCurrent": "31453184", + "MemoryDenyWriteExecute": "no", + "MemoryHigh": "infinity", + "MemoryLimit": "infinity", + "MemoryLow": "0", + "MemoryMax": "infinity", + "MemoryMin": "0", + "MemorySwapMax": "infinity", + "MountAPIVFS": "no", + "MountFlags": "", + "NFileDescriptorStore": "0", + "NRestarts": "0", + "NUMAMask": "", + "NUMAPolicy": "n/a", + "Names": "docker.service", + "NeedDaemonReload": "no", + "Nice": "0", + "NoNewPrivileges": "no", + "NonBlocking": "no", + "NotifyAccess": "main", + "OOMScoreAdjust": "-500", + "OnFailureJobMode": "replace", + "PermissionsStartOnly": "no", + "Perpetual": "no", + "PrivateDevices": "no", + "PrivateMounts": "no", + "PrivateNetwork": "no", + "PrivateTmp": "no", + "PrivateUsers": "no", + "ProtectControlGroups": "no", + "ProtectHome": "no", + "ProtectKernelModules": "no", + "ProtectKernelTunables": "no", + "ProtectSystem": "no", + "RefuseManualStart": "no", + "RefuseManualStop": "no", + "RemainAfterExit": "no", + "RemoveIPC": "no", + "Requires": "system.slice sysinit.target docker.socket", + "Restart": "always", + "RestartUSec": "2s", + "RestrictNamespaces": "no", + "RestrictRealtime": "no", + "RestrictSUIDSGID": "no", + "Result": "success", + "RootDirectoryStartOnly": "no", + "RuntimeDirectoryMode": "0755", + "RuntimeDirectoryPreserve": "no", + "RuntimeMaxUSec": "infinity", + "SameProcessGroup": "no", + "SecureBits": "0", + "SendSIGHUP": "no", + "SendSIGKILL": "yes", + "Slice": "system.slice", + "StandardError": "inherit", + "StandardInput": "null", + "StandardInputData": "", + "StandardOutput": "journal", + "StartLimitAction": "none", + "StartLimitBurst": "3", + "StartLimitIntervalUSec": "1min", + "StartupBlockIOWeight": "[not set]", + "StartupCPUShares": "[not set]", + "StartupCPUWeight": "[not set]", + "StartupIOWeight": "[not set]", + "StateChangeTimestamp": "Thu 2025-07-24 15:27:29 UTC", + "StateChangeTimestampMonotonic": "259862320", + "StateDirectoryMode": "0755", + "StatusErrno": "0", + "StopWhenUnneeded": "no", + "SubState": "running", + "SuccessAction": "none", + "SyslogFacility": "3", + "SyslogLevel": "6", + "SyslogLevelPrefix": "yes", + "SyslogPriority": "30", + "SystemCallErrorNumber": "0", + "TTYReset": "no", + "TTYVHangup": "no", + "TTYVTDisallocate": "no", + "TasksAccounting": "yes", + "TasksCurrent": "10", + "TasksMax": "infinity", + "TimeoutStartUSec": "infinity", + "TimeoutStopUSec": "1min 30s", + "TimerSlackNSec": "50000", + "Transient": "no", + "TriggeredBy": "docker.socket", + "Type": "notify", + "UID": "[not set]", + "UMask": "0022", + "UnitFilePreset": "disabled", + "UnitFileState": "enabled", + "UtmpMode": "init", + "WantedBy": "multi-user.target", + "Wants": "network-online.target containerd.service", + "WatchdogTimestamp": "Thu 2025-07-24 15:27:29 UTC", + "WatchdogTimestampMonotonic": "259862318", + "WatchdogUSec": "0" + } +} + +TASK [vdi_tool : Wait for Docker daemon to be ready] *************************** +ok: [localhost] => { + "changed": false, + "elapsed": 0, + "gid": 988, + "group": "docker", + "match_groupdict": {}, + "match_groups": [], + "mode": "0660", + "owner": "root", + "path": "/var/run/docker.sock", + "port": null, + "search_regex": null, + "size": 0, + "state": "file", + "uid": 0 +} + +TASK [vdi_tool : Test Docker connectivity] ************************************* +changed: [localhost] => { + "attempts": 1, + "changed": true, + "cmd": [ + "docker", + "version" + ], + "delta": "0:00:00.019833", + "end": "2025-07-24 15:31:01.979111", + "rc": 0, + "start": "2025-07-24 15:31:01.959278" +} + +STDOUT: + +Client: Docker Engine - Community + Version: 28.3.2 + API version: 1.51 + Go version: go1.24.5 + Git commit: 578ccf6 + Built: Wed Jul 9 16:15:41 2025 + OS/Arch: linux/amd64 + Context: default + +Server: Docker Engine - Community + Engine: + Version: 28.3.2 + API version: 1.51 (minimum version 1.24) + Go version: go1.24.5 + Git commit: e77ff99 + Built: Wed Jul 9 16:14:23 2025 + OS/Arch: linux/amd64 + Experimental: false + containerd: + Version: 1.7.27 + GitCommit: 05044ec0a9a75232cad458027ca83437aae3f4da + runc: + Version: 1.2.5 + GitCommit: v1.2.5-0-g59923ef + docker-init: + Version: 0.19.0 + GitCommit: de40ad0 + +TASK [vdi_tool : Enable pgcrypto extension for initdb] ************************* +changed: [localhost] => { + "changed": true, + "checksum": "a127ec70504ae55424b5bfdba85a53ce3fe2e710", + "dest": "/opt/guacamole-db/initdb/00-enable-pgcrypto.sql", + "gid": 0, + "group": "root", + "md5sum": "5759c18ba008c3af3b174a99dfedc20e", + "mode": "0644", + "owner": "root", + "size": 82, + "src": "/root/.ansible/tmp/ansible-tmp-1753371062.0273864-27523-222276405449246/source", + "state": "file", + "uid": 0 +} + +TASK [vdi_tool : Template per-user Guacamole user SQL] ************************* +changed: [localhost] => (item={'username': 'alice', 'port': 5901, 'password': 'xc61q4jU3mUCo84M', 'vncserver_password': '4JB1YHUQ', 'display_number': 1}) => { + "ansible_loop_var": "item", + "changed": true, + "checksum": "11e358d1c93056ca1f3197b5a7aa118878d69048", + "dest": "/opt/guacamole-db/initdb/02-alice_user.sql", + "gid": 0, + "group": "root", + "item": { + "display_number": 1, + "password": "xc61q4jU3mUCo84M", + "port": 5901, + "username": "alice", + "vncserver_password": "4JB1YHUQ" + }, + "md5sum": "2203ca51f69a089f76a44664596df631", + "mode": "0644", + "owner": "root", + "size": 734, + "src": "/root/.ansible/tmp/ansible-tmp-1753371062.378293-27545-63386938039965/source", + "state": "file", + "uid": 0 +} +changed: [localhost] => (item={'username': 'bob', 'port': 5902, 'secret_name': 'a-password-for-bob', 'password': 'B0bp4ssw0rd123', 'vncserver_password': '7DmxvFp2', 'display_number': 2}) => { + "ansible_loop_var": "item", + "changed": true, + "checksum": "9eabaa254800e4e435f8c70b918a4a6b428b6fbd", + "dest": "/opt/guacamole-db/initdb/02-bob_user.sql", + "gid": 0, + "group": "root", + "item": { + "display_number": 2, + "password": "B0bp4ssw0rd123", + "port": 5902, + "secret_name": "a-password-for-bob", + "username": "bob", + "vncserver_password": "7DmxvFp2" + }, + "md5sum": "73275e670b160e53c0acca30b52ae218", + "mode": "0644", + "owner": "root", + "size": 726, + "src": "/root/.ansible/tmp/ansible-tmp-1753371062.7223604-27545-271943085306166/source", + "state": "file", + "uid": 0 +} + +TASK [vdi_tool : Template per-user SQL for Guacamole connections] ************** +changed: [localhost] => (item={'username': 'alice', 'port': 5901, 'password': 'xc61q4jU3mUCo84M', 'vncserver_password': '4JB1YHUQ', 'display_number': 1}) => { + "ansible_loop_var": "item", + "changed": true, + "checksum": "a2f7d7fd0c1e596f84464e301312c00e0fa12a11", + "dest": "/opt/guacamole-db/initdb/03-alice_connection.sql", + "gid": 0, + "group": "root", + "item": { + "display_number": 1, + "password": "xc61q4jU3mUCo84M", + "port": 5901, + "username": "alice", + "vncserver_password": "4JB1YHUQ" + }, + "md5sum": "7df88ce1ea5f854682df295bfe2e301f", + "mode": "0644", + "owner": "root", + "size": 6583, + "src": "/root/.ansible/tmp/ansible-tmp-1753371063.081999-27587-195959994230920/source", + "state": "file", + "uid": 0 +} +changed: [localhost] => (item={'username': 'bob', 'port': 5902, 'secret_name': 'a-password-for-bob', 'password': 'B0bp4ssw0rd123', 'vncserver_password': '7DmxvFp2', 'display_number': 2}) => { + "ansible_loop_var": "item", + "changed": true, + "checksum": "291f5f0588de9e18befd6b849591ee5eab224333", + "dest": "/opt/guacamole-db/initdb/03-bob_connection.sql", + "gid": 0, + "group": "root", + "item": { + "display_number": 2, + "password": "B0bp4ssw0rd123", + "port": 5902, + "secret_name": "a-password-for-bob", + "username": "bob", + "vncserver_password": "7DmxvFp2" + }, + "md5sum": "1ae0fd1017d8d23d1acd289524cc7b5c", + "mode": "0644", + "owner": "root", + "size": 6547, + "src": "/root/.ansible/tmp/ansible-tmp-1753371063.4521995-27587-78729102064992/source", + "state": "file", + "uid": 0 +} + +TASK [vdi_tool : Template user cleanup SQL for removed users] ****************** +changed: [localhost] => { + "changed": true, + "checksum": "eed1f453966cbb3acf4a3266ca3b5998416a3706", + "dest": "/opt/guacamole-db/initdb/04-user_cleanup.sql", + "gid": 0, + "group": "root", + "md5sum": "b582c47aac06c808a4bb374e7dbad148", + "mode": "0644", + "owner": "root", + "size": 3166, + "src": "/root/.ansible/tmp/ansible-tmp-1753371063.8163276-27629-260203219362437/source", + "state": "file", + "uid": 0 +} + +TASK [vdi_tool : Generate combined initdb.sql] ********************************* +changed: [localhost] => { + "changed": true, + "cmd": "docker run --rm -v /opt/guacamole-db/initdb:/opt/guacamole-db/initdb guacamole/guacamole:latest /opt/guacamole/bin/initdb.sh --postgresql > /opt/guacamole-db/initdb/01-initdb.sql\n", + "delta": "0:00:12.451850", + "end": "2025-07-24 15:31:16.794061", + "rc": 0, + "start": "2025-07-24 15:31:04.342211" +} + +STDERR: + +Unable to find image 'guacamole/guacamole:latest' locally +latest: Pulling from guacamole/guacamole +32f112e3802c: Pulling fs layer +ea8f5ca39c1a: Pulling fs layer +a4f9b8cc7730: Pulling fs layer +4e45d9955da7: Pulling fs layer +59425f3d7529: Pulling fs layer +a875e361a596: Pulling fs layer +4f4fb700ef54: Pulling fs layer +cdbfaaae41bb: Pulling fs layer +f5cb881fc6db: Pulling fs layer +de7057472cfc: Pulling fs layer +7c43b83d031a: Pulling fs layer +8a4673b53887: Pulling fs layer +8630345c18ca: Pulling fs layer +a875e361a596: Waiting +4f4fb700ef54: Waiting +cdbfaaae41bb: Waiting +f5cb881fc6db: Waiting +de7057472cfc: Waiting +7c43b83d031a: Waiting +8a4673b53887: Waiting +8630345c18ca: Waiting +4e45d9955da7: Waiting +59425f3d7529: Waiting +ea8f5ca39c1a: Verifying Checksum +ea8f5ca39c1a: Download complete +32f112e3802c: Verifying Checksum +32f112e3802c: Download complete +4e45d9955da7: Download complete +59425f3d7529: Verifying Checksum +59425f3d7529: Download complete +a875e361a596: Verifying Checksum +a875e361a596: Download complete +4f4fb700ef54: Verifying Checksum +4f4fb700ef54: Download complete +cdbfaaae41bb: Verifying Checksum +cdbfaaae41bb: Download complete +de7057472cfc: Verifying Checksum +de7057472cfc: Download complete +f5cb881fc6db: Verifying Checksum +f5cb881fc6db: Download complete +32f112e3802c: Pull complete +8a4673b53887: Download complete +8630345c18ca: Download complete +ea8f5ca39c1a: Pull complete +a4f9b8cc7730: Verifying Checksum +a4f9b8cc7730: Download complete +7c43b83d031a: Verifying Checksum +7c43b83d031a: Download complete +a4f9b8cc7730: Pull complete +4e45d9955da7: Pull complete +59425f3d7529: Pull complete +a875e361a596: Pull complete +4f4fb700ef54: Pull complete +cdbfaaae41bb: Pull complete +f5cb881fc6db: Pull complete +de7057472cfc: Pull complete +7c43b83d031a: Pull complete +8a4673b53887: Pull complete +8630345c18ca: Pull complete +Digest: sha256:4cec231c1fe46367e1e3da80a12f0f59c48aebde9e6a97cdaf2a8fcf92376b4a +Status: Downloaded newer image for guacamole/guacamole:latest + +TASK [vdi_tool : Update guacadmin password hash in 01-initdb.sql] ************** +changed: [localhost] => { + "changed": true, + "rc": 0 +} + +MSG: + +1 replacements made + +TASK [vdi_tool : Remove salt from guacadmin in 01-initdb.sql] ****************** +changed: [localhost] => { + "changed": true, + "rc": 0 +} + +MSG: + +1 replacements made + +TASK [vdi_tool : Pull required Docker images] ********************************** +changed: [localhost] => (item=postgres:latest) => { + "ansible_loop_var": "item", + "changed": true, + "cmd": [ + "docker", + "pull", + "postgres:latest" + ], + "delta": "0:00:05.944704", + "end": "2025-07-24 15:31:23.396321", + "item": "postgres:latest", + "rc": 0, + "start": "2025-07-24 15:31:17.451617" +} + +STDOUT: + +latest: Pulling from library/postgres +59e22667830b: Pulling fs layer +c2922dd5c76b: Pulling fs layer +28b0f0abf5b3: Pulling fs layer +a623957f847d: Pulling fs layer +bfabbcdba989: Pulling fs layer +0e5bba37029a: Pulling fs layer +1ed9ff0b5160: Pulling fs layer +06b2fd76987c: Pulling fs layer +9e482b60495e: Pulling fs layer +4a43c57a903d: Pulling fs layer +dcce1c0b58a8: Pulling fs layer +38072b29d55e: Pulling fs layer +d98e5494f240: Pulling fs layer +017403a0b5e1: Pulling fs layer +0e5bba37029a: Waiting +1ed9ff0b5160: Waiting +06b2fd76987c: Waiting +9e482b60495e: Waiting +4a43c57a903d: Waiting +dcce1c0b58a8: Waiting +38072b29d55e: Waiting +d98e5494f240: Waiting +017403a0b5e1: Waiting +a623957f847d: Waiting +bfabbcdba989: Waiting +c2922dd5c76b: Verifying Checksum +c2922dd5c76b: Download complete +28b0f0abf5b3: Verifying Checksum +28b0f0abf5b3: Download complete +a623957f847d: Verifying Checksum +a623957f847d: Download complete +59e22667830b: Verifying Checksum +59e22667830b: Download complete +bfabbcdba989: Verifying Checksum +bfabbcdba989: Download complete +0e5bba37029a: Verifying Checksum +0e5bba37029a: Download complete +1ed9ff0b5160: Verifying Checksum +1ed9ff0b5160: Download complete +06b2fd76987c: Verifying Checksum +06b2fd76987c: Download complete +4a43c57a903d: Verifying Checksum +4a43c57a903d: Download complete +dcce1c0b58a8: Download complete +38072b29d55e: Download complete +d98e5494f240: Verifying Checksum +d98e5494f240: Download complete +59e22667830b: Pull complete +c2922dd5c76b: Pull complete +017403a0b5e1: Verifying Checksum +017403a0b5e1: Download complete +28b0f0abf5b3: Pull complete +a623957f847d: Pull complete +bfabbcdba989: Pull complete +0e5bba37029a: Pull complete +1ed9ff0b5160: Pull complete +06b2fd76987c: Pull complete +9e482b60495e: Verifying Checksum +9e482b60495e: Download complete +9e482b60495e: Pull complete +4a43c57a903d: Pull complete +dcce1c0b58a8: Pull complete +38072b29d55e: Pull complete +d98e5494f240: Pull complete +017403a0b5e1: Pull complete +Digest: sha256:4d89c904835259bc58876520e56267ca07a4ebd6a027f7814bbbf91b50d685be +Status: Downloaded newer image for postgres:latest +docker.io/library/postgres:latest +changed: [localhost] => (item=guacamole/guacd:latest) => { + "ansible_loop_var": "item", + "changed": true, + "cmd": [ + "docker", + "pull", + "guacamole/guacd:latest" + ], + "delta": "0:00:04.829341", + "end": "2025-07-24 15:31:28.396327", + "item": "guacamole/guacd:latest", + "rc": 0, + "start": "2025-07-24 15:31:23.566986" +} + +STDOUT: + +latest: Pulling from guacamole/guacd +44cf07d57ee4: Pulling fs layer +2c7959c3f8de: Pulling fs layer +d977524fd227: Pulling fs layer +26483f1a0b5c: Pulling fs layer +e2df05816112: Pulling fs layer +596f1cdf21db: Pulling fs layer +26483f1a0b5c: Waiting +596f1cdf21db: Waiting +e2df05816112: Waiting +2c7959c3f8de: Download complete +44cf07d57ee4: Download complete +44cf07d57ee4: Pull complete +26483f1a0b5c: Verifying Checksum +26483f1a0b5c: Download complete +e2df05816112: Verifying Checksum +e2df05816112: Download complete +2c7959c3f8de: Pull complete +596f1cdf21db: Verifying Checksum +596f1cdf21db: Download complete +d977524fd227: Download complete +d977524fd227: Pull complete +26483f1a0b5c: Pull complete +e2df05816112: Pull complete +596f1cdf21db: Pull complete +Digest: sha256:f70aa3356d1ae6ba7cff608309684c1736c7ae7b2b2e201e8e29281f2591227f +Status: Downloaded newer image for guacamole/guacd:latest +docker.io/guacamole/guacd:latest +changed: [localhost] => (item=guacamole/guacamole:latest) => { + "ansible_loop_var": "item", + "changed": true, + "cmd": [ + "docker", + "pull", + "guacamole/guacamole:latest" + ], + "delta": "0:00:00.359246", + "end": "2025-07-24 15:31:28.927638", + "item": "guacamole/guacamole:latest", + "rc": 0, + "start": "2025-07-24 15:31:28.568392" +} + +STDOUT: + +latest: Pulling from guacamole/guacamole +Digest: sha256:4cec231c1fe46367e1e3da80a12f0f59c48aebde9e6a97cdaf2a8fcf92376b4a +Status: Image is up to date for guacamole/guacamole:latest +docker.io/guacamole/guacamole:latest + +TASK [vdi_tool : Create a dedicated Docker network] **************************** +changed: [localhost] => { + "changed": true, + "cmd": [ + "docker", + "network", + "create", + "guac_net" + ], + "delta": "0:00:00.062475", + "end": "2025-07-24 15:31:29.181269", + "failed_when_result": false, + "rc": 0, + "start": "2025-07-24 15:31:29.118794" +} + +STDOUT: + +e0275c25416d70f991c99ebe7fb8ee8b8ca67ff902b5ea23f21f58461ec48bff + +TASK [vdi_tool : Start Guacamole PostgreSQL container] ************************* +changed: [localhost] => { + "changed": true, + "cmd": [ + "docker", + "run", + "-d", + "--name", + "guac_db", + "--network", + "guac_net", + "-e", + "POSTGRES_USER=guacamole_db", + "-e", + "POSTGRES_PASSWORD=H95L0yeLji7pzjDCOuemJDzEwoASP7Nq", + "-e", + "POSTGRES_DB=guacamole_db", + "-v", + "/opt/guacamole-db/data:/var/lib/postgresql/data", + "-v", + "/opt/guacamole-db/initdb:/docker-entrypoint-initdb.d", + "--restart", + "always", + "postgres:latest" + ], + "delta": "0:00:07.462040", + "end": "2025-07-24 15:31:36.839313", + "failed_when_result": false, + "rc": 0, + "start": "2025-07-24 15:31:29.377273" +} + +STDOUT: + +7f665322894de32870a3505d6da6ae46f83ea5d228a347e7ea1a71240c15015e + +TASK [vdi_tool : Wait for PostgreSQL container to be ready] ******************** +fatal: [localhost]: FAILED! => { + "changed": false, + "elapsed": 60 +} + +MSG: + +Timeout when waiting for 127.0.0.1:5432 +...ignoring + +TASK [vdi_tool : Wait for PostgreSQL via Docker exec] ************************** +changed: [localhost] => { + "attempts": 1, + "changed": true, + "cmd": [ + "docker", + "exec", + "guac_db", + "pg_isready", + "-U", + "guacamole_db" + ], + "delta": "0:00:00.088759", + "end": "2025-07-24 15:32:37.343504", + "rc": 0, + "start": "2025-07-24 15:32:37.254745" +} + +STDOUT: + +/var/run/postgresql:5432 - accepting connections + +TASK [vdi_tool : Check if database needs re-initialization due to user changes] *** +ok: [localhost] => { + "changed": false, + "cmd": [ + "docker", + "exec", + "guac_db", + "psql", + "-U", + "guacamole_db", + "-d", + "guacamole_db", + "-c", + "SELECT COUNT(*) FROM guacamole_entity WHERE type='USER' AND name != 'guacadmin';" + ], + "delta": "0:00:00.083631", + "end": "2025-07-24 15:32:37.618923", + "rc": 0, + "start": "2025-07-24 15:32:37.535292" +} + +STDOUT: + + count +------- + 2 +(1 row) + +TASK [vdi_tool : Debug user configuration status] ****************************** +ok: [localhost] => {} + +MSG: + +Database user check: count +------- + 2 +(1 row) +Current users in config: 2 +Lock file exists: True +User secrets hash changed: True +User secrets hash (stored): none +User secrets hash (current): 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 + + +TASK [vdi_tool : Check if user configuration has changed] ********************** +ok: [localhost] => { + "ansible_facts": { + "users_changed": true + }, + "changed": false +} + +TASK [vdi_tool : Debug user change detection] ********************************** +ok: [localhost] => {} + +MSG: + +Users changed: True + +TASK [vdi_tool : Stop Guacamole containers] ************************************ +fatal: [localhost]: FAILED! => { + "changed": true, + "cmd": [ + "docker", + "stop", + "guac_app", + "guacd" + ], + "delta": "0:00:00.013660", + "end": "2025-07-24 15:32:37.908714", + "rc": 1, + "start": "2025-07-24 15:32:37.895054" +} + +STDERR: + +Error response from daemon: No such container: guac_app +Error response from daemon: No such container: guacd + + +MSG: + +non-zero return code +...ignoring + +TASK [vdi_tool : Remove Guacamole containers] ********************************** +fatal: [localhost]: FAILED! => { + "changed": true, + "cmd": [ + "docker", + "rm", + "guac_app", + "guacd" + ], + "delta": "0:00:00.012838", + "end": "2025-07-24 15:32:38.111949", + "rc": 1, + "start": "2025-07-24 15:32:38.099111" +} + +STDERR: + +Error response from daemon: No such container: guac_app +Error response from daemon: No such container: guacd + + +MSG: + +non-zero return code +...ignoring + +TASK [vdi_tool : Stop and remove PostgreSQL container] ************************* +changed: [localhost] => { + "changed": true, + "cmd": [ + "docker", + "stop", + "guac_db" + ], + "delta": "0:00:00.134068", + "end": "2025-07-24 15:32:38.439413", + "rc": 0, + "start": "2025-07-24 15:32:38.305345" +} + +STDOUT: + +guac_db + +TASK [vdi_tool : Remove PostgreSQL container] ********************************** +changed: [localhost] => { + "changed": true, + "cmd": [ + "docker", + "rm", + "guac_db" + ], + "delta": "0:00:00.018178", + "end": "2025-07-24 15:32:38.655370", + "rc": 0, + "start": "2025-07-24 15:32:38.637192" +} + +STDOUT: + +guac_db + +TASK [vdi_tool : Backup existing database data] ******************************** +fatal: [localhost]: FAILED! => { + "changed": true, + "cmd": [ + "tar", + "-czf", + "/opt/guacamole-db/backup-$(date", + "+%Y%m%d-%H%M%S).tar.gz", + "-C", + "/opt/guacamole-db", + "data/" + ], + "delta": "0:00:00.927459", + "end": "2025-07-24 15:32:39.776351", + "rc": 2, + "start": "2025-07-24 15:32:38.848892" +} + +STDERR: + +tar: +%Y%m%d-%H%M%S).tar.gz: Cannot stat: No such file or directory +tar: Exiting with failure status due to previous errors + + +MSG: + +non-zero return code +...ignoring + +TASK [vdi_tool : Remove existing database data] ******************************** +changed: [localhost] => { + "changed": true, + "cmd": [ + "rm", + "-rf", + "/opt/guacamole-db/data/*" + ], + "delta": "0:00:00.002506", + "end": "2025-07-24 15:32:39.973183", + "rc": 0, + "start": "2025-07-24 15:32:39.970677" +} + +TASK [vdi_tool : Start Guacamole PostgreSQL container (re-initialized)] ******** +changed: [localhost] => { + "changed": true, + "cmd": [ + "docker", + "run", + "-d", + "--name", + "guac_db", + "--network", + "guac_net", + "-e", + "POSTGRES_USER=guacamole_db", + "-e", + "POSTGRES_PASSWORD=H95L0yeLji7pzjDCOuemJDzEwoASP7Nq", + "-e", + "POSTGRES_DB=guacamole_db", + "-v", + "/opt/guacamole-db/data:/var/lib/postgresql/data", + "-v", + "/opt/guacamole-db/initdb:/docker-entrypoint-initdb.d", + "--restart", + "always", + "postgres:latest" + ], + "delta": "0:00:00.214960", + "end": "2025-07-24 15:32:40.389033", + "failed_when_result": false, + "rc": 0, + "start": "2025-07-24 15:32:40.174073" +} + +STDOUT: + +f5711132145daa089efb4d0a763437ab81d2f13cdde1644b2472b45869dd4c81 + +TASK [vdi_tool : Wait for PostgreSQL to be ready after re-initialization] ****** +changed: [localhost] => { + "attempts": 1, + "changed": true, + "cmd": [ + "docker", + "exec", + "guac_db", + "pg_isready", + "-U", + "guacamole_db" + ], + "delta": "0:00:00.084157", + "end": "2025-07-24 15:32:40.671774", + "rc": 0, + "start": "2025-07-24 15:32:40.587617" +} + +STDOUT: + +/var/run/postgresql:5432 - accepting connections + +TASK [vdi_tool : Start guacd container] **************************************** +changed: [localhost] => { + "changed": true, + "cmd": [ + "docker", + "run", + "-d", + "--name", + "guacd", + "--network", + "guac_net", + "-p", + "4822:4822", + "--restart", + "always", + "guacamole/guacd:latest" + ], + "delta": "0:00:00.199585", + "end": "2025-07-24 15:32:41.064927", + "failed_when_result": false, + "rc": 0, + "start": "2025-07-24 15:32:40.865342" +} + +STDOUT: + +809f2c58fa7ebb5c36b56d14a529a6261f8fe3e30e1b6993d36b92d9cf905269 + +TASK [vdi_tool : Start Guacamole webapp container] ***************************** +changed: [localhost] => { + "changed": true, + "cmd": [ + "docker", + "run", + "-d", + "--name", + "guac_app", + "--network", + "guac_net", + "-p", + "8081:8080", + "-e", + "POSTGRES_HOSTNAME=guac_db", + "-e", + "POSTGRES_PORT=5432", + "-e", + "POSTGRES_DATABASE=guacamole_db", + "-e", + "POSTGRES_USER=guacamole_db", + "-e", + "POSTGRES_PASSWORD=H95L0yeLji7pzjDCOuemJDzEwoASP7Nq", + "-e", + "POSTGRES_AUTO_CREATE_ACCOUNTS=true", + "-e", + "GUACD_HOSTNAME=guacd", + "-e", + "GUACD_PORT=4822", + "--restart", + "always", + "guacamole/guacamole:latest" + ], + "delta": "0:00:00.203470", + "end": "2025-07-24 15:32:41.469743", + "failed_when_result": false, + "rc": 0, + "start": "2025-07-24 15:32:41.266273" +} + +STDOUT: + +671a01e330bf70d61b6a5aa26d573704e651f02a8b292abd5a8fb656bd885675 + +TASK [vdi_tool : Debug container status before waiting for Guacamole] ********** +ok: [localhost] => { + "changed": false, + "cmd": [ + "docker", + "ps" + ], + "delta": "0:00:00.015784", + "end": "2025-07-24 15:32:41.680190", + "rc": 0, + "start": "2025-07-24 15:32:41.664406" +} + +STDOUT: + +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +671a01e330bf guacamole/guacamole:latest "/opt/guacamole/bin/ā€¦" Less than a second ago Up Less than a second 0.0.0.0:8081->8080/tcp, [::]:8081->8080/tcp guac_app +809f2c58fa7e guacamole/guacd:latest "/opt/guacamole/entrā€¦" 1 second ago Up Less than a second (health: starting) 0.0.0.0:4822->4822/tcp, [::]:4822->4822/tcp guacd +f5711132145d postgres:latest "docker-entrypoint.sā€¦" 1 second ago Up 1 second 5432/tcp guac_db + +TASK [vdi_tool : Show container status] **************************************** +ok: [localhost] => {} + +MSG: + +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +671a01e330bf guacamole/guacamole:latest "/opt/guacamole/bin/ā€¦" Less than a second ago Up Less than a second 0.0.0.0:8081->8080/tcp, [::]:8081->8080/tcp guac_app +809f2c58fa7e guacamole/guacd:latest "/opt/guacamole/entrā€¦" 1 second ago Up Less than a second (health: starting) 0.0.0.0:4822->4822/tcp, [::]:4822->4822/tcp guacd +f5711132145d postgres:latest "docker-entrypoint.sā€¦" 1 second ago Up 1 second 5432/tcp guac_db + +TASK [vdi_tool : Wait for Guacamole HTTP endpoint] ***************************** +ok: [localhost] => { + "changed": false, + "elapsed": 2, + "match_groupdict": {}, + "match_groups": [], + "path": null, + "port": 8081, + "search_regex": null, + "state": "started" +} + +TASK [vdi_tool : Check Guacamole UI is up] ************************************* +ok: [localhost] => { + "accept_ranges": "bytes", + "attempts": 1, + "cache_control": "no-cache", + "changed": false, + "connection": "close", + "content_length": "2811", + "content_type": "text/html", + "cookies": {}, + "cookies_string": "", + "date": "Thu, 24 Jul 2025 15:32:45 GMT", + "elapsed": 0, + "etag": "W/\"2811-1753318742000\"", + "last_modified": "Thu, 24 Jul 2025 00:59:02 GMT", + "pragma": "no-cache", + "redirected": false, + "status": 200, + "url": "http://localhost:8081/guacamole/" +} + +MSG: + +OK (2811 bytes) + +TASK [vdi_tool : Obtain Guacamole API token to verify login] ******************* +ok: [localhost] => { + "attempts": 1, + "changed": false, + "connection": "close", + "content": "{\"authToken\":\"0D1A88B239CAB8F2BB454E77803481AA313179A99217A10E06963CAFFC219737\",\"username\":\"guacadmin\",\"dataSource\":\"postgresql\",\"availableDataSources\":[\"postgresql\",\"postgresql-shared\"]}", + "content_length": "187", + "content_type": "application/json", + "cookies": {}, + "cookies_string": "", + "date": "Thu, 24 Jul 2025 15:32:45 GMT", + "elapsed": 0, + "json": { + "authToken": "0D1A88B239CAB8F2BB454E77803481AA313179A99217A10E06963CAFFC219737", + "availableDataSources": [ + "postgresql", + "postgresql-shared" + ], + "dataSource": "postgresql", + "username": "guacadmin" + }, + "redirected": false, + "status": 200, + "url": "http://localhost:8081/guacamole/api/tokens" +} + +MSG: + +OK (187 bytes) + +TASK [vdi_tool : Remove bootstrap SQL files unless debug is enabled] *********** +skipping: [localhost] => { + "changed": false, + "false_condition": "not debug | default(false)", + "skip_reason": "Conditional result was False" +} + +TASK [vdi_tool : Install NoMachine] ******************************************** +skipping: [localhost] => { + "changed": false, + "false_condition": "vdi_tool | lower == 'nomachine'", + "skip_reason": "Conditional result was False" +} + +TASK [vdi_tool : Install Workspot] ********************************************* +skipping: [localhost] => { + "changed": false, + "false_condition": "vdi_tool | lower == 'workspot'", + "skip_reason": "Conditional result was False" +} + +TASK [vdi_tool : Set current role for lock manager completion] ***************** +ok: [localhost] => { + "ansible_facts": { + "current_role": "vdi_tool", + "role_completed": true + }, + "changed": false +} + +TASK [lock_manager : Debug create lock operation] ****************************** +ok: [localhost] => {} + +MSG: + +Creating/updating lock file for role: vdi_tool + +TASK [lock_manager : Check if lock file exists] ******************************** +ok: [localhost] => { + "changed": false, + "stat": { + "atime": 1753371032.8846393, + "attr_flags": "", + "attributes": [], + "block_size": 4096, + "blocks": 8, + "charset": "us-ascii", + "checksum": "542bc16243f6f0f5bd20bfbc7c7d0ae70dfc3965", + "ctime": 1753371032.652622, + "dev": 2050, + "device_type": 0, + "executable": false, + "exists": true, + "gid": 0, + "gr_name": "root", + "inode": 419489989, + "isblk": false, + "ischr": false, + "isdir": false, + "isfifo": false, + "isgid": false, + "islnk": false, + "isreg": true, + "issock": false, + "isuid": false, + "mimetype": "text/plain", + "mode": "0644", + "mtime": 1753371032.49061, + "nlink": 1, + "path": "/opt/vdi-setup/.vdi-lock.yaml", + "pw_name": "root", + "readable": true, + "rgrp": true, + "roth": true, + "rusr": true, + "size": 1066, + "uid": 0, + "version": "68461029", + "wgrp": false, + "woth": false, + "writeable": true, + "wusr": true, + "xgrp": false, + "xoth": false, + "xusr": false + } +} + +TASK [lock_manager : Debug lock file existence] ******************************** +ok: [localhost] => {} + +MSG: + +Lock file exists: True + +TASK [lock_manager : Create initial lock file structure] *********************** +skipping: [localhost] => { + "changed": false, + "false_condition": "not lock_file_stat.stat.exists", + "skip_reason": "Conditional result was False" +} + +TASK [lock_manager : Debug lock file creation] ********************************* +skipping: [localhost] => { + "false_condition": "debug | default(false) and not lock_file_stat.stat.exists" +} + +TASK [lock_manager : Load existing lock file data] ***************************** +ok: [localhost] => { + "ansible_facts": { + "vdi_setup_status": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": true, + "completed_at": "2025-07-24T15:30:30Z" + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:30Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:30Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "ansible_included_var_files": [ + "/opt/vdi-setup/.vdi-lock.yaml" + ], + "changed": false +} + +TASK [lock_manager : Get current lock data] ************************************ +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": true, + "completed_at": "2025-07-24T15:30:30Z" + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:30Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:30Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Update deployment hash] *********************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": true, + "completed_at": "2025-07-24T15:30:30Z" + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:30:30Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:30Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Get current timestamp] ************************************ +changed: [localhost] => { + "changed": true, + "cmd": [ + "date", + "-u", + "+%Y-%m-%dT%H:%M:%SZ" + ], + "delta": "0:00:00.002389", + "end": "2025-07-24 15:32:46.698118", + "rc": 0, + "start": "2025-07-24 15:32:46.695729" +} + +STDOUT: + +2025-07-24T15:32:46Z + +TASK [lock_manager : Update last updated timestamp] **************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": true, + "completed_at": "2025-07-24T15:30:30Z" + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:32:46Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:30:30Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Get current user secrets status] ************************** +ok: [localhost] => { + "ansible_facts": { + "current_user_secrets": { + "last_secret_check": "2025-07-24T15:30:30Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Create user secrets update] ******************************* +ok: [localhost] => { + "ansible_facts": { + "user_secrets_update": { + "last_secret_check": "2025-07-24T15:32:46Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" + } + }, + "changed": false +} + +TASK [lock_manager : Update user secrets hash] ********************************* +ok: [localhost] => { + "ansible_facts": { + "updated_user_secrets": { + "last_secret_check": "2025-07-24T15:32:46Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Apply user secrets update] ******************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": true, + "completed_at": "2025-07-24T15:30:30Z" + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:32:46Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:32:46Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Create role status entry] ********************************* +ok: [localhost] => { + "ansible_facts": { + "role_status_entry": { + "completed": true, + "completed_at": "2025-07-24T15:32:46Z" + } + }, + "changed": false +} + +TASK [lock_manager : Debug role status entry] ********************************** +ok: [localhost] => {} + +MSG: + +Role status entry for vdi_tool: + - completed: True + - completed_at: 2025-07-24T15:32:46Z + + +TASK [lock_manager : Debug current lock data before role update] *************** +ok: [localhost] => {} + +MSG: + +Current lock data before updating vdi_tool: + - current_lock_data: completed_roles: + base_os: + completed: true + completed_at: '2025-07-24T15:29:50Z' + lock_manager: + completed: true + completed_at: '2025-07-24T15:27:44Z' + secret_manager: + completed: true + completed_at: '2025-07-24T15:30:03Z' + user_provision: + completed: true + completed_at: '2025-07-24T15:30:16Z' + vdi_tool: + completed: false + vnc: + completed: true + completed_at: '2025-07-24T15:30:30Z' +created_at: '2025-07-24T15:27:41Z' +deployment_hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab +deployment_name: vdi-test-scott +force_rerun: false +last_updated: '2025-07-24T15:32:46Z' +lock_version: '1.0' +setup_status: configuring +user_management: + current_users: + - alice + - bob + previous_users: + - alice + - bob + removed_users: [] +user_secrets_status: + last_secret_check: '2025-07-24T15:32:46Z' + user_secrets_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 + users_updated: + - alice + - bob + + - current_completed_roles: base_os: + completed: true + completed_at: '2025-07-24T15:29:50Z' +lock_manager: + completed: true + completed_at: '2025-07-24T15:27:44Z' +secret_manager: + completed: true + completed_at: '2025-07-24T15:30:03Z' +user_provision: + completed: true + completed_at: '2025-07-24T15:30:16Z' +vdi_tool: + completed: false +vnc: + completed: true + completed_at: '2025-07-24T15:30:30Z' + + - role_completed: True + - current_role: vdi_tool + + +TASK [lock_manager : Get current completed roles] ****************************** +ok: [localhost] => { + "ansible_facts": { + "current_completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": false + }, + "vnc": { + "completed": true, + "completed_at": "2025-07-24T15:30:30Z" + } + } + }, + "changed": false +} + +TASK [lock_manager : Update completed roles] *********************************** +ok: [localhost] => { + "ansible_facts": { + "updated_completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": true, + "completed_at": "2025-07-24T15:32:46Z" + }, + "vnc": { + "completed": true, + "completed_at": "2025-07-24T15:30:30Z" + } + } + }, + "changed": false +} + +TASK [lock_manager : Update lock data with completed roles] ******************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": true, + "completed_at": "2025-07-24T15:32:46Z" + }, + "vnc": { + "completed": true, + "completed_at": "2025-07-24T15:30:30Z" + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:32:46Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:32:46Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Debug updated lock data after role update] **************** +ok: [localhost] => {} + +MSG: + +Updated lock data after updating vdi_tool: + - current_role: vdi_tool + - role_completed: True + - completed_roles: {'base_os': {'completed': True, 'completed_at': '2025-07-24T15:29:50Z'}, 'lock_manager': {'completed': True, 'completed_at': '2025-07-24T15:27:44Z'}, 'secret_manager': {'completed': True, 'completed_at': '2025-07-24T15:30:03Z'}, 'user_provision': {'completed': True, 'completed_at': '2025-07-24T15:30:16Z'}, 'vdi_tool': {'completed': True, 'completed_at': '2025-07-24T15:32:46Z'}, 'vnc': {'completed': True, 'completed_at': '2025-07-24T15:30:30Z'}} + + +TASK [lock_manager : Get current users updated list] *************************** +ok: [localhost] => { + "ansible_facts": { + "current_users_updated": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Get new usernames list] *********************************** +ok: [localhost] => { + "ansible_facts": { + "new_usernames": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Combine users updated lists] ****************************** +ok: [localhost] => { + "ansible_facts": { + "combined_users_updated": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Debug user deduplication] ********************************* +ok: [localhost] => {} + +MSG: + +User deduplication for vdi_tool: + - current_users_updated: ['alice', 'bob'] + - new_usernames: ['alice', 'bob'] + - combined_users_updated: ['alice', 'bob'] + + +TASK [lock_manager : Get current user secrets for list update] ***************** +ok: [localhost] => { + "ansible_facts": { + "current_user_secrets_for_list": { + "last_secret_check": "2025-07-24T15:32:46Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Update users updated list] ******************************** +ok: [localhost] => { + "ansible_facts": { + "updated_user_secrets_with_list": { + "last_secret_check": "2025-07-24T15:32:46Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + }, + "changed": false +} + +TASK [lock_manager : Apply users updated list] ********************************* +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": true, + "completed_at": "2025-07-24T15:32:46Z" + }, + "vnc": { + "completed": true, + "completed_at": "2025-07-24T15:30:30Z" + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:32:46Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:32:46Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Get current user management data] ************************* +ok: [localhost] => { + "ansible_facts": { + "current_user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + } + }, + "changed": false +} + +TASK [lock_manager : Get current usernames list] ******************************* +ok: [localhost] => { + "ansible_facts": { + "current_usernames": [ + "alice", + "bob" + ] + }, + "changed": false +} + +TASK [lock_manager : Update user management data] ****************************** +ok: [localhost] => { + "ansible_facts": { + "updated_user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + } + }, + "changed": false +} + +TASK [lock_manager : Apply user management update] ***************************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": true, + "completed_at": "2025-07-24T15:32:46Z" + }, + "vnc": { + "completed": true, + "completed_at": "2025-07-24T15:30:30Z" + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:32:46Z", + "lock_version": "1.0", + "setup_status": "configuring", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:32:46Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Debug user management] ************************************ +ok: [localhost] => {} + +MSG: + +User management for vdi_tool: + - current_users: ['alice', 'bob'] + - previous_users: ['alice', 'bob'] + - removed_users: [] + + +TASK [lock_manager : Check if all roles are completed] ************************* +ok: [localhost] => { + "ansible_facts": { + "all_roles_completed": true + }, + "changed": false +} + +TASK [lock_manager : Update setup status based on completion] ****************** +ok: [localhost] => { + "ansible_facts": { + "current_lock_data": { + "completed_roles": { + "base_os": { + "completed": true, + "completed_at": "2025-07-24T15:29:50Z" + }, + "lock_manager": { + "completed": true, + "completed_at": "2025-07-24T15:27:44Z" + }, + "secret_manager": { + "completed": true, + "completed_at": "2025-07-24T15:30:03Z" + }, + "user_provision": { + "completed": true, + "completed_at": "2025-07-24T15:30:16Z" + }, + "vdi_tool": { + "completed": true, + "completed_at": "2025-07-24T15:32:46Z" + }, + "vnc": { + "completed": true, + "completed_at": "2025-07-24T15:30:30Z" + } + }, + "created_at": "2025-07-24T15:27:41Z", + "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", + "deployment_name": "vdi-test-scott", + "force_rerun": false, + "last_updated": "2025-07-24T15:32:46Z", + "lock_version": "1.0", + "setup_status": "available", + "user_management": { + "current_users": [ + "alice", + "bob" + ], + "previous_users": [ + "alice", + "bob" + ], + "removed_users": [] + }, + "user_secrets_status": { + "last_secret_check": "2025-07-24T15:32:46Z", + "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", + "users_updated": [ + "alice", + "bob" + ] + } + } + }, + "changed": false +} + +TASK [lock_manager : Debug setup status update] ******************************** +ok: [localhost] => {} + +MSG: + +Setup status update: + - all_roles_completed: True + - setup_status: available + - completed_roles_count: 6 + - total_roles_count: 6 + + +TASK [lock_manager : Write updated lock file] ********************************** +changed: [localhost] => { + "changed": true, + "checksum": "fc55f0f0719e0f714591e3a785b23088e4370d6b", + "dest": "/opt/vdi-setup/.vdi-lock.yaml", + "gid": 0, + "group": "root", + "md5sum": "023467ccf639a5ab5102d3857dac0237", + "mode": "0644", + "owner": "root", + "size": 1106, + "src": "/root/.ansible/tmp/ansible-tmp-1753371167.8426328-29425-109328817609540/source", + "state": "file", + "uid": 0 +} + +TASK [lock_manager : Debug VM metadata update] ********************************* +ok: [localhost] => {} + +MSG: + +VM metadata update info: + - hostname: vdi-test-scott-rocky-0 + - zone: us-central1-a + - lock_file_changed: True + + +TASK [lock_manager : Get lock file content as base64] ************************** +changed: [localhost] => { + "changed": true, + "cmd": "cat /opt/vdi-setup/.vdi-lock.yaml | base64 -w 0", + "delta": "0:00:00.003891", + "end": "2025-07-24 15:32:48.398036", + "rc": 0, + "start": "2025-07-24 15:32:48.394145" +} + +STDOUT: + +dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiB0cnVlCiAgICAgIGNvbXBsZXRlZF9hdDogJzIwMjUtMDctMjRUMTU6MzA6MTZaJwogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjMyOjQ2WicKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiB0cnVlCiAgICAgIGNvbXBsZXRlZF9hdDogJzIwMjUtMDctMjRUMTU6MzA6MzBaJwogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNTozMjo0NlonCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogYXZhaWxhYmxlCiAgdXNlcl9tYW5hZ2VtZW50OgogICAgY3VycmVudF91c2VyczoKICAgIC0gYWxpY2UKICAgIC0gYm9iCiAgICBwcmV2aW91c191c2VyczoKICAgIC0gYWxpY2UKICAgIC0gYm9iCiAgICByZW1vdmVkX3VzZXJzOiBbXQogIHVzZXJfc2VjcmV0c19zdGF0dXM6CiAgICBsYXN0X3NlY3JldF9jaGVjazogJzIwMjUtMDctMjRUMTU6MzI6NDZaJwogICAgdXNlcl9zZWNyZXRzX2hhc2g6IDZhNmEwMmQzZDA5OTMzY2EyODJhZTJiMDIzMzYzNDNjOWQ3MzcwMWU0NWJkODE0NzVmYmQ5MmRjZjlkMmYzYjYKICAgIHVzZXJzX3VwZGF0ZWQ6CiAgICAtIGFsaWNlCiAgICAtIGJvYgo= + +TASK [lock_manager : Add VDI lock file content to VM metadata] ***************** +changed: [localhost] => { + "changed": true, + "cmd": [ + "gcloud", + "compute", + "instances", + "add-metadata", + "vdi-test-scott-rocky-0", + "--metadata", + "vdi-lock-content=dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiB0cnVlCiAgICAgIGNvbXBsZXRlZF9hdDogJzIwMjUtMDctMjRUMTU6MzA6MTZaJwogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjMyOjQ2WicKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiB0cnVlCiAgICAgIGNvbXBsZXRlZF9hdDogJzIwMjUtMDctMjRUMTU6MzA6MzBaJwogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNTozMjo0NlonCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogYXZhaWxhYmxlCiAgdXNlcl9tYW5hZ2VtZW50OgogICAgY3VycmVudF91c2VyczoKICAgIC0gYWxpY2UKICAgIC0gYm9iCiAgICBwcmV2aW91c191c2VyczoKICAgIC0gYWxpY2UKICAgIC0gYm9iCiAgICByZW1vdmVkX3VzZXJzOiBbXQogIHVzZXJfc2VjcmV0c19zdGF0dXM6CiAgICBsYXN0X3NlY3JldF9jaGVjazogJzIwMjUtMDctMjRUMTU6MzI6NDZaJwogICAgdXNlcl9zZWNyZXRzX2hhc2g6IDZhNmEwMmQzZDA5OTMzY2EyODJhZTJiMDIzMzYzNDNjOWQ3MzcwMWU0NWJkODE0NzVmYmQ5MmRjZjlkMmYzYjYKICAgIHVzZXJzX3VwZGF0ZWQ6CiAgICAtIGFsaWNlCiAgICAtIGJvYgo=", + "--zone=us-central1-a" + ], + "delta": "0:00:04.026047", + "end": "2025-07-24 15:32:52.625449", + "failed_when_result": false, + "rc": 0, + "start": "2025-07-24 15:32:48.599402" +} + +STDERR: + +Updated [https://www.googleapis.com/compute/v1/projects/hpc-discovery-external/zones/us-central1-a/instances/vdi-test-scott-rocky-0]. + +TASK [lock_manager : Debug lock file write] ************************************ +ok: [localhost] => {} + +MSG: + +Lock file written: True + +TASK [lock_manager : Verify lock file integrity] ******************************* +ok: [localhost] => { + "changed": false, + "cmd": [ + "python3", + "-c", + "import yaml; yaml.safe_load(open('/opt/vdi-setup/.vdi-lock.yaml'))" + ], + "delta": "0:00:00.040826", + "end": "2025-07-24 15:32:52.906476", + "rc": 0, + "start": "2025-07-24 15:32:52.865650" +} + +TASK [lock_manager : Debug lock file validation] ******************************* +ok: [localhost] => {} + +MSG: + +Lock file validation: True + +TASK [lock_manager : Check if VDI monitor service exists] ********************** +ok: [localhost] => { + "changed": false, + "stat": { + "atime": 1753370874.1548011, + "attr_flags": "", + "attributes": [], + "block_size": 4096, + "blocks": 8, + "charset": "us-ascii", + "checksum": "f6de22707f8d9248014f679a2f1930451e2bf635", + "ctime": 1753370872.6256874, + "dev": 2050, + "device_type": 0, + "executable": false, + "exists": true, + "gid": 0, + "gr_name": "root", + "inode": 285214140, + "isblk": false, + "ischr": false, + "isdir": false, + "isfifo": false, + "isgid": false, + "islnk": false, + "isreg": true, + "issock": false, + "isuid": false, + "mimetype": "text/plain", + "mode": "0644", + "mtime": 1753370872.4766762, + "nlink": 1, + "path": "/etc/systemd/system/vdi-monitor.service", + "pw_name": "root", + "readable": true, + "rgrp": true, + "roth": true, + "rusr": true, + "size": 283, + "uid": 0, + "version": "1338188335", + "wgrp": false, + "woth": false, + "writeable": true, + "wusr": true, + "xgrp": false, + "xoth": false, + "xusr": false + } +} + +TASK [lock_manager : Debug VDI monitor setup conditions] *********************** +ok: [localhost] => {} + +MSG: + +VDI monitor setup conditions: + - current_role: vdi_tool + - vdi_monitor_service_exists: True + - lock_file_creation.changed: False + - will_setup_monitor: False + + +TASK [lock_manager : Setup VDI monitoring service (first time only)] *********** +skipping: [localhost] => { + "changed": false, + "false_condition": "not vdi_monitor_service_stat.stat.exists", + "skip_reason": "Conditional result was False" +} + +TASK [lock_manager : Fail if lock file is invalid] ***************************** +skipping: [localhost] => { + "changed": false, + "false_condition": "lock_file_validation.rc != 0", + "skip_reason": "Conditional result was False" +} + +PLAY RECAP ********************************************************************* +localhost : ok=551 changed=75 unreachable=0 failed=0 skipped=68 rescued=0 ignored=6 + +Thu Jul 24 15:32:53 +0000 2025 Info [1870]: === install.yaml-f44d finished with exit_code=0 === +Thu Jul 24 15:32:53 +0000 2025 Info [1535]: === passed_startup_script.sh-22f5 finished with exit_code=0 === +[root@vdi-test-scott-rocky-0 ~]# \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/main.tf b/community/modules/scripts/vdi-setup/main.tf index 38aec83f2c..2a338fa919 100644 --- a/community/modules/scripts/vdi-setup/main.tf +++ b/community/modules/scripts/vdi-setup/main.tf @@ -24,6 +24,10 @@ data "archive_file" "roles_tar" { output_path = "${path.module}/roles.tar.gz" } +resource "random_id" "resource_name_suffix" { + byte_length = 4 +} + # Generate vars content using templatefile locals { vdi_vars_content = templatefile("${path.module}/templates/vars.yaml.tftpl", { @@ -36,6 +40,9 @@ locals { vdi_webapp_port = var.vdi_webapp_port vdi_resolution = var.vdi_resolution vdi_users = var.vdi_users + debug = var.debug + vdi_bucket_name = local.bucket_name + zone = var.zone }) } @@ -84,18 +91,16 @@ locals { type = "ansible-local" content = templatefile("${path.module}/templates/install.yaml.tftpl", { - roles = ["base_os", "secret_manager", "user_provision", "vnc", "vdi_tool"], + roles = ["lock_manager", "base_os", "secret_manager", "user_provision", "vnc", "vdi_tool"], } ) destination = "/tmp/vdi/install.yaml" - # Debug mode: - # args = "--extra-vars @/tmp/vdi/vars.yaml -v --extra-vars debug=true" - args = "--extra-vars @/tmp/vdi/vars.yaml" + args = var.debug ? "--extra-vars @/tmp/vdi/vars.yaml -v --extra-vars debug=true" : "--extra-vars @/tmp/vdi/vars.yaml" }, # Todo: another runner here to delete /tmp/vdi afterwards? ] - bucket_name = "${substr(var.deployment_name, 0, 39)}-vdi-scripts-${substr(md5(var.deployment_name), 0, 8)}" + bucket_name = "${substr(var.deployment_name, 0, 39)}-vdi-scripts-${random_id.resource_name_suffix.hex}" } # Bucket to stage runners diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml index 982bae6c51..1a68dc133c 100644 --- a/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml @@ -13,34 +13,72 @@ # limitations under the License. --- +- name: DEBUG About to import lock_manager for base_os + debug: + msg: "About to import lock_manager for base_os" + +# Check if this role should run +- name: Set current role for lock manager check + ansible.builtin.set_fact: + current_role: "base_os" + +- name: Check if base_os role should run + ansible.builtin.import_role: + name: lock_manager + tasks_from: check_lock + register: lock_check_result + +- name: DEBUG Finished import_role for base_os + debug: + msg: "Finished import_role for base_os" + +- name: Set role_should_run from lock check result + ansible.builtin.set_fact: + role_should_run: "{{ lock_check_result.ansible_facts.role_should_run | default(true) }}" + +# Skip all tasks if role should not run +- name: Skip base_os tasks if role should not run + ansible.builtin.debug: + msg: "Skipping base_os role - already completed or not needed" + when: not role_should_run + - name: Gather distribution facts ansible.builtin.setup: gather_subset: platform + when: role_should_run # GPU / vWS driver setup - name: Check if NVIDIA GPU is present ansible.builtin.shell: lspci | grep -i nvidia register: nvidia_gpu_present failed_when: false + when: role_should_run # Include distribution-specific tasks - name: Include Ubuntu-specific tasks ansible.builtin.include_tasks: ubuntu.yaml - when: ansible_facts.distribution == 'Ubuntu' + when: + - ansible_facts.distribution == 'Ubuntu' + - role_should_run - name: Include Rocky Linux specific tasks ansible.builtin.include_tasks: rocky.yaml - when: ansible_facts.distribution == 'Rocky' + when: + - ansible_facts.distribution == 'Rocky' + - role_should_run - name: Include Debian-specific tasks ansible.builtin.include_tasks: debian.yaml - when: ansible_facts.distribution == 'Debian' + when: + - ansible_facts.distribution == 'Debian' + - role_should_run # Common tasks for all distributions - name: Ensure requests package is installed pip: name: requests state: present + when: role_should_run - name: Ensure Python & tooling are installed ansible.builtin.package: @@ -49,8 +87,24 @@ - python3-pip - jq state: present + when: role_should_run # GPU / vWS driver setup - name: Provision GPU setup ansible.builtin.include_tasks: gpu_driver_setup.yaml - when: nvidia_gpu_present.rc == 0 + when: + - nvidia_gpu_present.rc == 0 + - role_should_run + +# Mark role as completed +- name: Set current role for lock manager completion + ansible.builtin.set_fact: + current_role: "base_os" + role_completed: true + when: role_should_run + +- name: Mark base_os role as completed + ansible.builtin.import_role: + name: lock_manager + tasks_from: create_lock + when: role_should_run diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/check_lock.yaml b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/check_lock.yaml new file mode 100644 index 0000000000..86807a9470 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/check_lock.yaml @@ -0,0 +1,203 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Check if lock file exists + ansible.builtin.stat: + path: /opt/vdi-setup/.vdi-lock.yaml + register: lock_file_stat + +- name: Debug lock file existence + ansible.builtin.debug: + msg: "Lock file exists: {{ lock_file_stat.stat.exists }}" + when: debug | default(false) + +- name: Load existing lock file + ansible.builtin.include_vars: + file: /opt/vdi-setup/.vdi-lock.yaml + when: lock_file_stat.stat.exists + register: lock_file_load + +- name: Debug lock file load result + ansible.builtin.debug: + msg: "Lock file loaded successfully: {{ lock_file_load is defined }}" + when: debug | default(false) and lock_file_stat.stat.exists + +- name: Calculate current deployment hash + ansible.builtin.set_fact: + current_deployment_hash: >- + {{ + (deployment_name | string + + project_id | string + + vdi_tool | string + + vnc_flavor | string + + user_provision | string + + vdi_webapp_port | string + + vdi_resolution | string + + (vdi_users | to_json)) | hash('sha256') + }} + +- name: Debug deployment hash calculation + ansible.builtin.debug: + msg: | + Current deployment hash: {{ current_deployment_hash }} + Deployment components: + - deployment_name: {{ deployment_name }} + - project_id: {{ project_id }} + - vdi_tool: {{ vdi_tool }} + - vnc_flavor: {{ vnc_flavor }} + - user_provision: {{ user_provision }} + - vdi_webapp_port: {{ vdi_webapp_port }} + - vdi_resolution: {{ vdi_resolution }} + - vdi_users count: {{ vdi_users | length }} + when: debug | default(false) + +- name: Calculate current user secrets hash + ansible.builtin.set_fact: + current_user_secrets_hash: "{{ (vdi_users | to_json | sort) | hash('sha256') }}" + +- name: Debug user secrets hash calculation + ansible.builtin.debug: + msg: | + Current user secrets hash: {{ current_user_secrets_hash }} + User configuration components: + - vdi_users (blueprint): {{ vdi_users | to_json }} + when: debug | default(false) + +- name: Set default values for missing lock file + ansible.builtin.set_fact: + vdi_setup_status: + deployment_name: "{{ deployment_name }}" + deployment_hash: "none" + lock_version: "1.0" + created_at: "{{ ansible_date_time.iso8601 }}" + last_updated: "{{ ansible_date_time.iso8601 }}" + force_rerun: false + setup_status: "configuring" + completed_roles: + base_os: + completed: false + lock_manager: + completed: false + secret_manager: + completed: false + user_provision: + completed: false + vnc: + completed: false + vdi_tool: + completed: false + user_secrets_status: + last_secret_check: "{{ ansible_date_time.iso8601 }}" + user_secrets_hash: "none" + users_updated: [] + user_management: + current_users: [] + previous_users: [] + removed_users: [] + when: not lock_file_stat.stat.exists + +- name: Debug lock file status + ansible.builtin.debug: + msg: | + Lock file status: + - exists: {{ lock_file_stat.stat.exists }} + - deployment_hash: {{ vdi_setup_status.deployment_hash }} + - user_secrets_hash: {{ vdi_setup_status.user_secrets_status.user_secrets_hash }} + - force_rerun: {{ vdi_setup_status.force_rerun }} + when: debug | default(false) + +- name: Check if lock file exists + ansible.builtin.set_fact: + lock_file_exists: "{{ lock_file_stat.stat.exists }}" + +- name: Check if current role is completed + ansible.builtin.set_fact: + current_role_completed: "{{ vdi_setup_status.completed_roles[current_role | default(ansible_role_name | basename)].completed | default(false) }}" + +- name: Check if deployment hash matches + ansible.builtin.set_fact: + deployment_hash_matches: "{{ vdi_setup_status.deployment_hash == current_deployment_hash }}" + +- name: Check if force rerun is enabled + ansible.builtin.set_fact: + force_rerun_enabled: "{{ vdi_setup_status.force_rerun | default(false) }}" + +- name: Determine if role should run + ansible.builtin.set_fact: + role_should_run: "{{ not lock_file_exists or not current_role_completed or not deployment_hash_matches or force_rerun_enabled }}" + +- name: Debug role should run calculation + ansible.builtin.debug: + msg: | + Role should run calculation for {{ current_role | default(ansible_role_name | basename) }}: + - not lock_file_exists: {{ not lock_file_exists }} + - not current_role_completed: {{ not current_role_completed }} + - not deployment_hash_matches: {{ not deployment_hash_matches }} + - force_rerun_enabled: {{ force_rerun_enabled }} + - role_should_run: {{ not lock_file_exists or not current_role_completed or not deployment_hash_matches or force_rerun_enabled }} + when: debug | default(false) + +- name: Debug role execution decision + ansible.builtin.debug: + msg: | + Role execution decision for {{ current_role | default(ansible_role_name | basename) }}: + - current_role: {{ current_role | default('not set') }} + - ansible_role_name: {{ ansible_role_name }} + - role_should_run: {{ role_should_run }} + - lock_file_exists: {{ lock_file_exists }} + - current_role_completed: {{ current_role_completed }} + - deployment_hash_matches: {{ deployment_hash_matches }} + - force_rerun_enabled: {{ force_rerun_enabled }} + when: debug | default(false) + +- name: Check if user configuration has changed + ansible.builtin.set_fact: + user_config_changed: "{{ not lock_file_stat.stat.exists or vdi_setup_status.user_secrets_status.user_secrets_hash != current_user_secrets_hash }}" + +- name: Set user secrets changed flag (will be updated by secret_manager role) + ansible.builtin.set_fact: + user_secrets_changed: "{{ user_config_changed }}" + +- name: Debug user secrets change detection + ansible.builtin.debug: + msg: | + User configuration change detection: + - user_config_changed: {{ user_config_changed }} + - user_secrets_changed: {{ user_secrets_changed }} + - stored_hash: {{ vdi_setup_status.user_secrets_status.user_secrets_hash }} + - current_hash: {{ current_user_secrets_hash }} + when: debug | default(false) + +- name: Set lock check result + ansible.builtin.set_fact: + lock_check_result: + role_should_run: "{{ role_should_run }}" + current_deployment_hash: "{{ current_deployment_hash }}" + existing_deployment_hash: "{{ vdi_setup_status.deployment_hash | default('none') }}" + user_secrets_changed: "{{ user_secrets_changed }}" + current_user_secrets_hash: "{{ current_user_secrets_hash }}" + existing_user_secrets_hash: "{{ vdi_setup_status.user_secrets_status.user_secrets_hash | default('none') }}" + force_rerun: "{{ vdi_setup_status.force_rerun | default(false) }}" + +- name: Debug final lock check result + ansible.builtin.debug: + msg: | + Final lock check result: + - role_should_run: {{ lock_check_result.role_should_run }} + - deployment_hash_changed: {{ lock_check_result.current_deployment_hash != lock_check_result.existing_deployment_hash }} + - user_secrets_changed: {{ lock_check_result.user_secrets_changed }} + - force_rerun: {{ lock_check_result.force_rerun }} + when: debug | default(false) + \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/cleanup_lock.yaml b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/cleanup_lock.yaml new file mode 100644 index 0000000000..c6b1eec63a --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/cleanup_lock.yaml @@ -0,0 +1,29 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Debug cleanup lock operation + ansible.builtin.debug: + msg: "Cleaning up lock file" + +- name: Remove lock file + ansible.builtin.file: + path: /opt/vdi-setup/.vdi-lock.yaml + state: absent + register: lock_file_removal + +- name: Debug lock file removal + ansible.builtin.debug: + msg: "Lock file removed: {{ lock_file_removal.changed }}" + when: debug | default(false) diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/create_lock.yaml b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/create_lock.yaml new file mode 100644 index 0000000000..c464599e32 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/create_lock.yaml @@ -0,0 +1,308 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Debug create lock operation + ansible.builtin.debug: + msg: "Creating/updating lock file for role: {{ current_role }}" + +- name: Check if lock file exists + ansible.builtin.stat: + path: /opt/vdi-setup/.vdi-lock.yaml + register: lock_file_stat + +- name: Debug lock file existence + ansible.builtin.debug: + msg: "Lock file exists: {{ lock_file_stat.stat.exists }}" + when: debug | default(false) + +- name: Create initial lock file structure + ansible.builtin.template: + src: lock_file.yaml.j2 + dest: /opt/vdi-setup/.vdi-lock.yaml + mode: '0644' + when: not lock_file_stat.stat.exists + register: lock_file_creation + +- name: Debug lock file creation + ansible.builtin.debug: + msg: "Lock file created: {{ lock_file_creation.changed }}" + when: debug | default(false) and not lock_file_stat.stat.exists + +- name: Load existing lock file data + ansible.builtin.include_vars: + file: /opt/vdi-setup/.vdi-lock.yaml + when: lock_file_stat.stat.exists or lock_file_creation.changed + register: existing_lock_data + +- name: Get current lock data + ansible.builtin.set_fact: + current_lock_data: "{{ existing_lock_data.ansible_facts.vdi_setup_status | default({}) }}" + when: lock_file_stat.stat.exists or lock_file_creation.changed + +- name: Update deployment hash + ansible.builtin.set_fact: + current_lock_data: "{{ current_lock_data | combine({'deployment_hash': current_deployment_hash}) }}" + when: lock_file_stat.stat.exists or lock_file_creation.changed + +- name: Get current timestamp + ansible.builtin.command: date -u +%Y-%m-%dT%H:%M:%SZ + register: current_timestamp + when: lock_file_stat.stat.exists or lock_file_creation.changed + +- name: Update last updated timestamp + ansible.builtin.set_fact: + current_lock_data: "{{ current_lock_data | combine({'last_updated': current_timestamp.stdout}) }}" + when: lock_file_stat.stat.exists or lock_file_creation.changed + +- name: Get current user secrets status + ansible.builtin.set_fact: + current_user_secrets: "{{ current_lock_data.user_secrets_status | default({}) }}" + when: lock_file_stat.stat.exists or lock_file_creation.changed + +- name: Create user secrets update + ansible.builtin.set_fact: + user_secrets_update: + user_secrets_hash: "{{ current_user_secrets_hash }}" + last_secret_check: "{{ current_timestamp.stdout }}" + when: lock_file_stat.stat.exists or lock_file_creation.changed + +- name: Update user secrets hash + ansible.builtin.set_fact: + updated_user_secrets: "{{ current_user_secrets | combine(user_secrets_update) }}" + when: lock_file_stat.stat.exists or lock_file_creation.changed + +- name: Apply user secrets update + ansible.builtin.set_fact: + current_lock_data: "{{ current_lock_data | combine({'user_secrets_status': updated_user_secrets}) }}" + when: lock_file_stat.stat.exists or lock_file_creation.changed + +- name: Create role status entry + ansible.builtin.set_fact: + role_status_entry: + completed: "{{ role_completed | default(false) }}" + completed_at: "{{ current_timestamp.stdout }}" + when: lock_file_stat.stat.exists or lock_file_creation.changed + +- name: Debug role status entry + ansible.builtin.debug: + msg: | + Role status entry for {{ current_role | default(ansible_role_name | basename) }}: + - completed: {{ role_completed | default(false) }} + - completed_at: {{ current_timestamp.stdout }} + when: debug | default(false) and (lock_file_stat.stat.exists or lock_file_creation.changed) + +- name: Debug current lock data before role update + ansible.builtin.debug: + msg: | + Current lock data before updating {{ current_role }}: + - current_lock_data: {{ current_lock_data | to_nice_yaml }} + - current_completed_roles: {{ current_lock_data.completed_roles | default({}) | to_nice_yaml }} + - role_completed: {{ role_completed }} + - current_role: {{ current_role }} + when: debug | default(false) and (lock_file_stat.stat.exists or lock_file_creation.changed) and role_completed is defined + +- name: Get current completed roles + ansible.builtin.set_fact: + current_completed_roles: "{{ current_lock_data.completed_roles | default({}) }}" + when: lock_file_stat.stat.exists or lock_file_creation.changed + +- name: Update completed roles + ansible.builtin.set_fact: + updated_completed_roles: "{{ current_completed_roles | combine({current_role | default(ansible_role_name | basename): role_status_entry}) }}" + when: lock_file_stat.stat.exists or lock_file_creation.changed + +- name: Update lock data with completed roles + ansible.builtin.set_fact: + current_lock_data: "{{ current_lock_data | combine({'completed_roles': updated_completed_roles}) }}" + when: lock_file_stat.stat.exists or lock_file_creation.changed + +- name: Debug updated lock data after role update + ansible.builtin.debug: + msg: | + Updated lock data after updating {{ current_role | default(ansible_role_name | basename) }}: + - current_role: {{ current_role | default(ansible_role_name | basename) }} + - role_completed: {{ role_completed | default(false) }} + - completed_roles: {{ updated_completed_roles }} + when: debug | default(false) and (lock_file_stat.stat.exists or lock_file_creation.changed) + +- name: Get current users updated list + ansible.builtin.set_fact: + current_users_updated: "{{ current_lock_data.user_secrets_status.users_updated | default([]) }}" + when: (lock_file_stat.stat.exists or lock_file_creation.changed) and user_secrets_changed + +- name: Get new usernames list + ansible.builtin.set_fact: + new_usernames: "{{ vdi_users | map(attribute='username') | list }}" + when: (lock_file_stat.stat.exists or lock_file_creation.changed) and user_secrets_changed + +- name: Combine users updated lists + ansible.builtin.set_fact: + combined_users_updated: "{{ (current_users_updated + new_usernames) | unique | list }}" + when: (lock_file_stat.stat.exists or lock_file_creation.changed) and user_secrets_changed + +- name: Debug user deduplication + ansible.builtin.debug: + msg: | + User deduplication for {{ current_role }}: + - current_users_updated: {{ current_users_updated }} + - new_usernames: {{ new_usernames }} + - combined_users_updated: {{ combined_users_updated }} + when: debug | default(false) and (lock_file_stat.stat.exists or lock_file_creation.changed) and user_secrets_changed + +- name: Get current user secrets for list update + ansible.builtin.set_fact: + current_user_secrets_for_list: "{{ current_lock_data.user_secrets_status }}" + when: (lock_file_stat.stat.exists or lock_file_creation.changed) and user_secrets_changed + +- name: Update users updated list + ansible.builtin.set_fact: + updated_user_secrets_with_list: "{{ current_user_secrets_for_list | combine({'users_updated': combined_users_updated}) }}" + when: (lock_file_stat.stat.exists or lock_file_creation.changed) and user_secrets_changed + +- name: Apply users updated list + ansible.builtin.set_fact: + current_lock_data: "{{ current_lock_data | combine({'user_secrets_status': updated_user_secrets_with_list}) }}" + when: (lock_file_stat.stat.exists or lock_file_creation.changed) and user_secrets_changed + +- name: Get current user management data + ansible.builtin.set_fact: + current_user_management: "{{ current_lock_data.user_management | default({}) }}" + when: lock_file_stat.stat.exists or lock_file_creation.changed + +- name: Get current usernames list + ansible.builtin.set_fact: + current_usernames: "{{ vdi_users | map(attribute='username') | list }}" + when: lock_file_stat.stat.exists or lock_file_creation.changed + +- name: Update user management data + ansible.builtin.set_fact: + updated_user_management: + previous_users: "{{ current_user_management.current_users | default([]) }}" + current_users: "{{ current_usernames }}" + removed_users: "{{ (current_user_management.current_users | default([])) | difference(current_usernames) }}" + when: lock_file_stat.stat.exists or lock_file_creation.changed + +- name: Apply user management update + ansible.builtin.set_fact: + current_lock_data: "{{ current_lock_data | combine({'user_management': updated_user_management}) }}" + when: lock_file_stat.stat.exists or lock_file_creation.changed + +- name: Debug user management + ansible.builtin.debug: + msg: | + User management for {{ current_role }}: + - current_users: {{ current_usernames }} + - previous_users: {{ current_user_management.current_users | default([]) }} + - removed_users: {{ (current_user_management.current_users | default([])) | difference(current_usernames) }} + when: debug | default(false) and (lock_file_stat.stat.exists or lock_file_creation.changed) + +- name: Check if all roles are completed + ansible.builtin.set_fact: + all_roles_completed: "{{ updated_completed_roles.values() | selectattr('completed', 'equalto', true) | list | length == updated_completed_roles.keys() | list | length }}" + when: lock_file_stat.stat.exists or lock_file_creation.changed + +- name: Update setup status based on completion + ansible.builtin.set_fact: + current_lock_data: "{{ current_lock_data | combine({'setup_status': 'available' if all_roles_completed else 'configuring'}) }}" + when: lock_file_stat.stat.exists or lock_file_creation.changed + +- name: Debug setup status update + ansible.builtin.debug: + msg: | + Setup status update: + - all_roles_completed: {{ all_roles_completed | default(false) }} + - setup_status: {{ 'available' if all_roles_completed else 'configuring' }} + - completed_roles_count: {{ updated_completed_roles.values() | selectattr('completed', 'equalto', true) | list | length }} + - total_roles_count: {{ updated_completed_roles.keys() | list | length }} + when: debug | default(false) and (lock_file_stat.stat.exists or lock_file_creation.changed) + +- name: Write updated lock file + ansible.builtin.copy: + content: | + vdi_setup_status: + {{ current_lock_data | to_nice_yaml(indent=2) | indent(2, first=true) }} + dest: /opt/vdi-setup/.vdi-lock.yaml + mode: '0644' + when: lock_file_stat.stat.exists or lock_file_creation.changed + register: lock_file_write + +- name: Debug VM metadata update + ansible.builtin.debug: + msg: | + VM metadata update info: + - hostname: {{ ansible_hostname }} + - zone: {{ zone }} + - lock_file_changed: {{ lock_file_write.changed }} + when: debug | default(false) + +- name: Get lock file content as base64 + ansible.builtin.shell: cat /opt/vdi-setup/.vdi-lock.yaml | base64 -w 0 + register: lock_file_base64 + when: lock_file_write.changed + +- name: Add VDI lock file content to VM metadata + ansible.builtin.command: > + gcloud compute instances add-metadata {{ ansible_hostname }} + --metadata vdi-lock-content="{{ lock_file_base64.stdout }}" + --zone={{ zone }} + when: + - lock_file_write.changed + - lock_file_base64.rc == 0 + register: metadata_update + changed_when: metadata_update.rc == 0 + failed_when: + - metadata_update.rc != 0 + - "'already exists' not in metadata_update.stderr | default('')" + +- name: Debug lock file write + ansible.builtin.debug: + msg: "Lock file written: {{ lock_file_write.changed }}" + when: debug | default(false) + +- name: Verify lock file integrity + ansible.builtin.command: python3 -c "import yaml; yaml.safe_load(open('/opt/vdi-setup/.vdi-lock.yaml'))" + register: lock_file_validation + changed_when: false + +- name: Debug lock file validation + ansible.builtin.debug: + msg: "Lock file validation: {{ lock_file_validation.rc == 0 }}" + when: debug | default(false) + +- name: Check if VDI monitor service exists + ansible.builtin.stat: + path: /etc/systemd/system/vdi-monitor.service + register: vdi_monitor_service_stat + +- name: Debug VDI monitor setup conditions + ansible.builtin.debug: + msg: | + VDI monitor setup conditions: + - current_role: {{ current_role }} + - vdi_monitor_service_exists: {{ vdi_monitor_service_stat.stat.exists }} + - lock_file_creation.changed: {{ lock_file_creation.changed | default(false) }} + - will_setup_monitor: {{ (not vdi_monitor_service_stat.stat.exists) and (lock_file_creation.changed | default(false)) }} + when: debug | default(false) + +- name: Setup VDI monitoring service (first time only) + ansible.builtin.include_tasks: monitor_setup.yaml + when: + - not vdi_monitor_service_stat.stat.exists + - lock_file_creation.changed + +- name: Fail if lock file is invalid + ansible.builtin.fail: + msg: "Lock file validation failed: {{ lock_file_validation.stderr }}" + when: lock_file_validation.rc != 0 diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/main.yaml new file mode 100644 index 0000000000..1f4a1fa6d5 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/main.yaml @@ -0,0 +1,60 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Set current role (only if not already set) + ansible.builtin.set_fact: + current_role: "{{ current_role | default('lock_manager') }}" + +- name: Debug lock manager role execution + ansible.builtin.debug: + msg: "Lock manager role started - operation: {{ lock_operation | default('check') }} for role: {{ current_role }}" + +- name: Ensure lock directory exists + ansible.builtin.file: + path: /opt/vdi-setup + state: directory + mode: '0755' + register: lock_dir_creation + +- name: Debug lock directory status + ansible.builtin.debug: + msg: "Lock directory created: {{ lock_dir_creation.changed }}" + when: debug | default(false) + +- name: Include check lock tasks + ansible.builtin.include_tasks: check_lock.yaml + when: lock_operation | default('check') == 'check' + +- name: Include create lock tasks + ansible.builtin.include_tasks: create_lock.yaml + when: lock_operation | default('check') == 'create' + +- name: Include cleanup lock tasks + ansible.builtin.include_tasks: cleanup_lock.yaml + when: lock_operation | default('check') == 'cleanup' + +- name: Debug lock manager role completion + ansible.builtin.debug: + msg: "Lock manager role completed - operation: {{ lock_operation | default('check') }}" + +# Mark lock_manager role as completed when running as a standalone role +- name: Mark lock_manager role as completed + ansible.builtin.include_tasks: create_lock.yaml + vars: + current_role: "lock_manager" + role_completed: true + when: + - current_role == "lock_manager" + - lock_operation | default('check') == 'check' diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/monitor_setup.yaml b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/monitor_setup.yaml new file mode 100644 index 0000000000..cacb9573b3 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/monitor_setup.yaml @@ -0,0 +1,73 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +- name: Create VDI monitor script directory + ansible.builtin.file: + path: /usr/local/bin + state: directory + mode: '0755' + +- name: Copy VDI monitor script + ansible.builtin.template: + src: vdi-monitor.sh.j2 + dest: /usr/local/bin/vdi-monitor.sh + mode: '0755' + owner: root + group: root + +- name: Copy VDI monitor systemd service + ansible.builtin.template: + src: vdi-monitor.service.j2 + dest: /etc/systemd/system/vdi-monitor.service + mode: '0644' + owner: root + group: root + +- name: Create VDI monitor log directory + ansible.builtin.file: + path: /var/log + state: directory + mode: '0755' + +- name: Create VDI monitor log files + ansible.builtin.file: + path: "{{ item }}" + state: touch + mode: '0644' + owner: root + group: root + loop: + - /var/log/vdi-monitor.log + - /var/log/ansible-vdi-reconfig.log + +- name: Reload systemd daemon + ansible.builtin.systemd: + daemon_reload: yes + +- name: Enable VDI monitor service + ansible.builtin.systemd: + name: vdi-monitor + enabled: yes + state: started + +- name: Debug VDI monitor setup + ansible.builtin.debug: + msg: | + VDI monitor service setup complete: + - Service: vdi-monitor.service + - Script: /usr/local/bin/vdi-monitor.sh + - Logs: /var/log/vdi-monitor.log, /var/log/ansible-vdi-reconfig.log + - Service enabled and started + when: debug | default(false) diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 b/community/modules/scripts/vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 new file mode 100644 index 0000000000..91f0253cf2 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 @@ -0,0 +1,35 @@ +# VDI Setup Lock File +# Generated on {{ ansible_date_time.iso8601 }} + +vdi_setup_status: + deployment_name: "{{ deployment_name }}" + deployment_hash: "{{ current_deployment_hash }}" + lock_version: "1.0" + created_at: "{{ ansible_date_time.iso8601 }}" + last_updated: "{{ ansible_date_time.iso8601 }}" + force_rerun: false + setup_status: "configuring" # configuring, available, error + + completed_roles: + base_os: + completed: false + lock_manager: + completed: false + secret_manager: + completed: false + user_provision: + completed: false + vnc: + completed: false + vdi_tool: + completed: false + + user_secrets_status: + last_secret_check: "{{ ansible_date_time.iso8601 }}" + user_secrets_hash: "{{ current_user_secrets_hash }}" + users_updated: [] + + user_management: + current_users: [] + previous_users: [] + removed_users: [] diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/templates/vdi-monitor.service.j2 b/community/modules/scripts/vdi-setup/roles/lock_manager/templates/vdi-monitor.service.j2 new file mode 100644 index 0000000000..58d9449b6b --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/lock_manager/templates/vdi-monitor.service.j2 @@ -0,0 +1,17 @@ +[Unit] +Description=VDI Configuration Monitor +After=network.target +Wants=network.target + +[Service] +Type=simple +User=root +Group=root +ExecStart=/usr/local/bin/vdi-monitor.sh +Restart=always +RestartSec=30 +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/templates/vdi-monitor.sh.j2 b/community/modules/scripts/vdi-setup/roles/lock_manager/templates/vdi-monitor.sh.j2 new file mode 100644 index 0000000000..6a8bf54614 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/lock_manager/templates/vdi-monitor.sh.j2 @@ -0,0 +1,184 @@ +#!/bin/bash + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# VDI Configuration Monitor +# Monitors the VDI lock file for changes and triggers re-configuration + +LOCK_FILE="/opt/vdi-setup/.vdi-lock.yaml" +CONFIG_HASH_FILE="/opt/vdi-setup/.config-hash" +MONITOR_LOG="/var/log/vdi-monitor.log" +ANSIBLE_LOG="/var/log/ansible-vdi-reconfig.log" +BUCKET_NAME="{{ vdi_bucket_name }}" +VDI_SETUP_DIR="/opt/vdi-setup" + +# Function to log messages +log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$MONITOR_LOG" +} + +# Function to sync files from bucket +sync_from_bucket() { + log_message "Syncing files from bucket: gs://$BUCKET_NAME" + + # Create temporary directory for sync + local temp_dir=$(mktemp -d) + + # Download files from bucket + if gsutil -m rsync -r "gs://$BUCKET_NAME/" "$temp_dir/"; then + log_message "Successfully downloaded files from bucket" + + # Copy files to VDI setup directory + if [[ -d "$temp_dir/roles" ]]; then + cp -r "$temp_dir/roles" "$VDI_SETUP_DIR/" + log_message "Updated roles directory" + fi + + if [[ -f "$temp_dir/install.yaml" ]]; then + cp "$temp_dir/install.yaml" "$VDI_SETUP_DIR/" + log_message "Updated install.yaml" + fi + + if [[ -f "$temp_dir/vars.yaml" ]]; then + cp "$temp_dir/vars.yaml" "$VDI_SETUP_DIR/" + log_message "Updated vars.yaml" + fi + + # Clean up temporary directory + rm -rf "$temp_dir" + return 0 + else + log_message "ERROR: Failed to sync files from bucket" + rm -rf "$temp_dir" + return 1 + fi +} + +# Function to get current configuration hash +get_config_hash() { + if [[ -f "$LOCK_FILE" ]]; then + # Extract deployment_hash and user_secrets_hash from lock file + deployment_hash=$(grep "deployment_hash:" "$LOCK_FILE" | awk '{print $2}' | tr -d '"') + user_secrets_hash=$(grep "user_secrets_hash:" "$LOCK_FILE" | awk '{print $2}' | tr -d '"') + echo "${deployment_hash}-${user_secrets_hash}" + else + echo "no-lock-file" + fi +} + +# Function to check if configuration has changed +check_config_changed() { + local current_hash=$(get_config_hash) + local stored_hash="" + + if [[ -f "$CONFIG_HASH_FILE" ]]; then + stored_hash=$(cat "$CONFIG_HASH_FILE") + fi + + if [[ "$current_hash" != "$stored_hash" ]]; then + log_message "Configuration change detected: $stored_hash -> $current_hash" + return 0 + else + return 1 + fi +} + +# Function to update stored hash +update_stored_hash() { + local current_hash=$(get_config_hash) + echo "$current_hash" > "$CONFIG_HASH_FILE" + log_message "Updated stored hash: $current_hash" +} + +# Function to trigger VDI re-configuration +trigger_reconfig() { + log_message "Triggering VDI re-configuration" + + # First, sync latest files from bucket + if ! sync_from_bucket; then + log_message "WARNING: Failed to sync from bucket, proceeding with local files" + fi + + # Change to VDI setup directory + cd "$VDI_SETUP_DIR" || { + log_message "ERROR: Cannot change to $VDI_SETUP_DIR" + return 1 + } + + # Run Ansible playbook with re-configuration + ansible-playbook install.yaml \ + --connection=local \ + --inventory=localhost, \ + --limit=localhost \ + --extra-vars="force_rerun=true" \ + --extra-vars="debug=true" \ + >> "$ANSIBLE_LOG" 2>&1 + + local exit_code=$? + if [[ $exit_code -eq 0 ]]; then + log_message "VDI re-configuration completed successfully" + update_stored_hash + else + log_message "ERROR: VDI re-configuration failed with exit code $exit_code" + log_message "Check $ANSIBLE_LOG for details" + fi + + return $exit_code +} + +# Main monitoring loop +main() { + log_message "VDI Configuration Monitor started" + + # Initialize stored hash if not exists + if [[ ! -f "$CONFIG_HASH_FILE" ]]; then + update_stored_hash + fi + + # Main monitoring loop + while true; do + if check_config_changed; then + trigger_reconfig + fi + + # Sleep for 60 seconds before next check + sleep 60 + done +} + +# Test mode function +test_mode() { + log_message "VDI Configuration Monitor - Test Mode" + log_message "Testing bucket sync functionality..." + + if sync_from_bucket; then + log_message "SUCCESS: Bucket sync test passed" + exit 0 + else + log_message "FAILED: Bucket sync test failed" + exit 1 + fi +} + +# Handle script termination +trap 'log_message "VDI Configuration Monitor stopped"; exit 0' SIGTERM SIGINT + +# Check for test mode +if [[ "${1:-}" == "--test" ]]; then + test_mode +fi + +# Start monitoring +main diff --git a/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml index 50eab08ffa..e7b5f69ce0 100644 --- a/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml @@ -13,90 +13,129 @@ # limitations under the License. --- -# General secret handling -- name: Generate random database password +# Check if this role should run +- name: Set current role for lock manager check ansible.builtin.set_fact: - database_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}" + current_role: "secret_manager" -- name: Generate random webapp admin password - ansible.builtin.set_fact: - webapp_admin_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}" +- name: Check if secret_manager role should run + ansible.builtin.import_role: + name: lock_manager + tasks_from: check_lock + register: lock_check_result -- name: Compute Webapp admin password hash to save to Webapp server +- name: Set role_should_run from lock check result ansible.builtin.set_fact: - webapp_admin_hash: "{{ webapp_admin_password | hash('sha256') | upper }}" - -- name: Get access token for Secret Manager API - ansible.builtin.uri: - url: "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" - method: GET - headers: - Metadata-Flavor: Google - return_content: true - register: access_token_result - -- name: Create a GCP secret for webapp server login - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets?secretId=webapp-server-password-{{ deployment_name }}" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - replication: - automatic: {} - status_code: [200, 409] # 409 = already exists - register: webapp_secret_create - -- name: Add webapp password version to secret - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/webapp-server-password-{{ deployment_name }}:addVersion" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - payload: - data: "{{ webapp_admin_password | b64encode }}" - register: webapp_secret - -- name: Create a GCP secret for database password - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets?secretId=db-password-{{ deployment_name }}" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - replication: - automatic: {} - status_code: [200, 409] # 409 = already exists - register: database_secret_create - -- name: Add database password version to secret - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/db-password-{{ deployment_name }}:addVersion" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - payload: - data: "{{ database_password | b64encode }}" - register: database_secret - -# Per-user secret handling -- name: Generate per-user secrets and enrich list - include_tasks: user_secret_tasks.yaml - loop: "{{ vdi_users }}" - loop_control: - loop_var: item - -- name: Set enriched vdi_users_updated variable + role_should_run: "{{ lock_check_result.ansible_facts.role_should_run | default(true) }}" + +# Skip all tasks if role should not run +- name: Skip secret_manager tasks if role should not run + ansible.builtin.debug: + msg: "Skipping secret_manager role - already completed or not needed" + when: not role_should_run + +# Run secret manager tasks only if needed +- name: Run secret manager tasks + block: + # General secret handling + - name: Generate random database password + ansible.builtin.set_fact: + database_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}" + + - name: Generate random webapp admin password + ansible.builtin.set_fact: + webapp_admin_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}" + + - name: Compute Webapp admin password hash to save to Webapp server + ansible.builtin.set_fact: + webapp_admin_hash: "{{ webapp_admin_password | hash('sha256') | upper }}" + + - name: Get access token for Secret Manager API + ansible.builtin.uri: + url: "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" + method: GET + headers: + Metadata-Flavor: Google + return_content: true + register: access_token_result + + - name: Create a GCP secret for webapp server login + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets?secretId=webapp-server-password-{{ deployment_name }}" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + replication: + automatic: {} + status_code: [200, 409] # 409 = already exists + register: webapp_secret_create + + - name: Add webapp password version to secret + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/webapp-server-password-{{ deployment_name }}:addVersion" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + payload: + data: "{{ webapp_admin_password | b64encode }}" + register: webapp_secret + + - name: Create a GCP secret for database password + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets?secretId=db-password-{{ deployment_name }}" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + replication: + automatic: {} + status_code: [200, 409] # 409 = already exists + register: database_secret_create + + - name: Add database password version to secret + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/db-password-{{ deployment_name }}:addVersion" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + payload: + data: "{{ database_password | b64encode }}" + register: database_secret + + # Per-user secret handling + - name: Generate per-user secrets and enrich list + include_tasks: user_secret_tasks.yaml + loop: "{{ vdi_users }}" + loop_control: + loop_var: item + + - name: Set enriched vdi_users_updated variable + ansible.builtin.set_fact: + vdi_users_updated: "{{ vdi_users_updated }}" + when: vdi_users_updated is defined + + when: role_should_run + +# Mark role as completed +- name: Set current role for lock manager completion ansible.builtin.set_fact: - vdi_users_updated: "{{ vdi_users_updated }}" - when: vdi_users_updated is defined + current_role: "secret_manager" + role_completed: true + when: role_should_run + +- name: Mark secret_manager role as completed + ansible.builtin.import_role: + name: lock_manager + tasks_from: create_lock + when: role_should_run diff --git a/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml b/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml index 272e7163fd..aafe3954bf 100644 --- a/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml +++ b/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml @@ -27,6 +27,19 @@ {%- elif item.password is defined -%}password {%- else -%}generate{%- endif -%} +- name: Clean password source value + ansible.builtin.set_fact: + password_source: "{{ password_source | trim }}" + +- name: Debug password source for {{ item.username }} + ansible.builtin.debug: + msg: | + Password source for {{ item.username }}: + - password_source: "{{ password_source }}" + - secret_name: {{ item.secret_name | default('not set') }} + - password: {{ item.password | default('not set') }} + when: debug | default(false) + - name: Fetch existing Secret Manager password for user {{ item.username }} ansible.builtin.uri: url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets/{{ item.secret_name }}/versions/latest:access" @@ -37,13 +50,38 @@ return_content: true register: sm_result failed_when: false - when: password_source | trim == "secret" + when: password_source == "secret" + +- name: Debug Secret Manager fetch result for {{ item.username }} + ansible.builtin.debug: + msg: | + Secret Manager fetch for {{ item.username }}: + - status: {{ sm_result.status }} + - success: {{ sm_result.status == 200 }} + - user_config_changed: {{ user_config_changed }} + - password_source: {{ password_source }} + when: debug | default(false) and password_source == "secret" - name: Set VDI user password from Secret Manager ansible.builtin.set_fact: vdiuser_password: "{{ sm_result.json.payload.data | b64decode }}" when: password_source == "secret" and sm_result.status == 200 +- name: Check if password has actually changed for {{ item.username }} + ansible.builtin.set_fact: + password_actually_changed: "{{ user_config_changed or (password_source == 'secret' and sm_result.status == 200 and vdiuser_password != item.password | default('')) }}" + when: password_source == "secret" and sm_result.status == 200 + +- name: Debug password change detection for {{ item.username }} + ansible.builtin.debug: + msg: | + Password change detection for {{ item.username }}: + - user_config_changed: {{ user_config_changed }} + - password_source: {{ password_source }} + - sm_result_status: {{ sm_result.status }} + - password_actually_changed: {{ password_actually_changed | default(false) }} + when: debug | default(false) and password_source == "secret" + - name: Generate random password for user {{ item.username }} ansible.builtin.set_fact: vdiuser_password: "{{ lookup('password', '/dev/null length=16 chars=ascii_letters,digits') }}" @@ -108,6 +146,17 @@ data: "{{ vdiuser_password | b64encode }}" when: password_source == "generate" +- name: Set VNC configuration for user + ansible.builtin.set_fact: + vnc_config: >- + {{ + (vnc_flavor | lower in ['tightvnc', 'tigervnc']) + | ternary({ + 'vncserver_password': lookup('community.general.random_string', length=8, special=false), + 'display_number': item.port | int - 5900 + }, {}) + }} + - name: Build enriched user list ansible.builtin.set_fact: vdi_users_updated: >- @@ -116,12 +165,6 @@ + [ item | combine({ 'password': vdiuser_password }) - | combine( - (vnc_flavor | lower in ['tightvnc', 'tigervnc']) - | ternary({ - 'vncserver_password': lookup('community.general.random_string', length=8, special=false), - 'display_number': item.port | int - 5900 - }, {}) - ) + | combine(vnc_config) ] }} diff --git a/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml index 8eabc84b2c..12063aa9a3 100644 --- a/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml +++ b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml @@ -13,6 +13,34 @@ # limitations under the License. --- +- name: Check if user exists + ansible.builtin.getent: + database: passwd + key: "{{ item.username }}" + register: user_exists_check + ignore_errors: true + when: debug | default(false) + +- name: Set user exists status for debug + ansible.builtin.set_fact: + user_exists_status: false + when: debug | default(false) and user_exists_check is failed + +- name: Set user exists status for debug (success case) + ansible.builtin.set_fact: + user_exists_status: "{{ user_exists_check.ansible_facts.getent_passwd is defined }}" + when: debug | default(false) and user_exists_check is not failed + +- name: Debug user provision for {{ item.username }} + ansible.builtin.debug: + msg: | + Provisioning user {{ item.username }}: + - user_secrets_changed: {{ user_secrets_changed }} + - password_actually_changed: {{ password_actually_changed | default(false) }} + - password_update_mode: {{ 'always' if password_actually_changed | default(user_secrets_changed) else 'on_create' }} + - user_exists: {{ user_exists_status | default(false) }} + when: debug | default(false) + - name: Provision VDI users (custom or generated passwords) block: @@ -25,7 +53,7 @@ create_home: yes append: true password: "{{ item.password | password_hash('sha512') }}" - update_password: on_create # sets pw if user is newly created + update_password: "{{ 'always' if password_actually_changed | default(user_secrets_changed) else 'on_create' }}" # SSH key setup - name: Ensure .ssh directory for {{ item.username }} @@ -49,3 +77,25 @@ user: "{{ item.username }}" key: "{{ ssh_key.public_key }}" state: present + +- name: Get users to remove from lock file + ansible.builtin.set_fact: + users_to_remove: "{{ vdi_setup_status.user_management.removed_users | default([]) }}" + when: lock_file_stat.stat.exists + +- name: Debug local user cleanup + ansible.builtin.debug: + msg: | + Local user cleanup: + - users_to_remove: {{ users_to_remove }} + when: debug | default(false) and lock_file_stat.stat.exists + +- name: Remove local user accounts + ansible.builtin.user: + name: "{{ item }}" + state: absent + remove: yes + loop: "{{ users_to_remove }}" + when: + - lock_file_stat.stat.exists + - users_to_remove | length > 0 diff --git a/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml index a8dc4f652c..bbeb26975c 100644 --- a/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml @@ -13,28 +13,67 @@ # limitations under the License. --- -- name: Ensure 'wheel' group is present - group: - name: wheel - state: present - -- name: Ensure VDI user group exists - ansible.builtin.group: - name: "{{ vdi_user_group }}" - state: present - -# Run localā€user provisioning if selected -- name: Provision local users - ansible.builtin.include_tasks: local_users.yaml - loop: "{{ vdi_users_updated }}" - loop_control: - loop_var: item - when: user_provision | lower == 'local_users' - -# Run OS Login tasks if selected -- name: Provision OS Login users - ansible.builtin.include_tasks: os_login.yaml - loop: "{{ vdi_users_updated }}" - loop_control: - loop_var: item - when: user_provision | lower == 'os_login' +# Check if this role should run +- name: Set current role for lock manager check + ansible.builtin.set_fact: + current_role: "user_provision" + +- name: Check if user_provision role should run + ansible.builtin.import_role: + name: lock_manager + tasks_from: check_lock + register: lock_check_result + +- name: Set role_should_run from lock check result + ansible.builtin.set_fact: + role_should_run: "{{ lock_check_result.ansible_facts.role_should_run | default(true) }}" + +# Skip all tasks if role should not run +- name: Skip user_provision tasks if role should not run + ansible.builtin.debug: + msg: "Skipping user_provision role - already completed or not needed" + when: not role_should_run + +# Run user provision tasks only if needed +- name: Run user provision tasks + block: + - name: Ensure 'wheel' group is present + group: + name: wheel + state: present + + - name: Ensure VDI user group exists + ansible.builtin.group: + name: "{{ vdi_user_group }}" + state: present + + # Run localā€user provisioning if selected + - name: Provision local users + ansible.builtin.include_tasks: local_users.yaml + loop: "{{ vdi_users_updated }}" + loop_control: + loop_var: item + when: user_provision | lower == 'local_users' + + # Run OS Login tasks if selected + - name: Provision OS Login users + ansible.builtin.include_tasks: os_login.yaml + loop: "{{ vdi_users_updated }}" + loop_control: + loop_var: item + when: user_provision | lower == 'os_login' + + when: role_should_run + +# Mark role as completed +- name: Set current role for lock manager completion + ansible.builtin.set_fact: + current_role: "user_provision" + role_completed: true + when: role_should_run + +- name: Mark user_provision role as completed + ansible.builtin.import_role: + name: lock_manager + tasks_from: create_lock + when: role_should_run diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml index e023dafe9a..c5f6c994ab 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml @@ -58,7 +58,7 @@ - name: Template per-user Guacamole user SQL ansible.builtin.template: - src: user_bootstrap.sql.j2 + src: guacamole/user_bootstrap.sql.j2 dest: "/opt/guacamole-db/initdb/02-{{ item.username }}_user.sql" owner: root group: root @@ -72,7 +72,7 @@ - name: Template per-user SQL for Guacamole connections ansible.builtin.template: - src: connection_bootstrap.sql.j2 + src: guacamole/connection_bootstrap.sql.j2 dest: "/opt/guacamole-db/initdb/03-{{ item.username }}_connection.sql" owner: root group: root @@ -85,6 +85,16 @@ username: "{{ item.username }}" vnc_password: "{{ item.vncserver_password }}" +- name: Template user cleanup SQL for removed users + ansible.builtin.template: + src: guacamole/user_cleanup.sql.j2 + dest: "/opt/guacamole-db/initdb/04-user_cleanup.sql" + owner: root + group: root + mode: '0644' + vars: + vdi_users_updated: "{{ vdi_users_updated }}" + - name: Generate combined initdb.sql ansible.builtin.shell: | docker run --rm \ @@ -151,6 +161,83 @@ retries: 30 delay: 2 +- name: Check if database needs re-initialization due to user changes + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db + -c "SELECT COUNT(*) FROM guacamole_entity WHERE type='USER' AND name != 'guacadmin';" + register: existing_users_check + changed_when: false + +- name: Debug user configuration status + ansible.builtin.debug: + msg: | + Database user check: {{ existing_users_check.stdout.strip() }} + Current users in config: {{ vdi_users_updated | length }} + Lock file exists: {{ lock_file_stat.stat.exists }} + User secrets hash changed: {{ vdi_setup_status.user_secrets_status.user_secrets_hash != current_user_secrets_hash if lock_file_stat.stat.exists else 'N/A' }} + User secrets hash (stored): {{ vdi_setup_status.user_secrets_status.user_secrets_hash if lock_file_stat.stat.exists else 'N/A' }} + User secrets hash (current): {{ current_user_secrets_hash }} + +- name: Check if user configuration has changed + ansible.builtin.set_fact: + users_changed: "{{ not lock_file_stat.stat.exists or vdi_setup_status.user_secrets_status.user_secrets_hash != current_user_secrets_hash }}" + +- name: Debug user change detection + ansible.builtin.debug: + msg: "Users changed: {{ users_changed }}" + +- name: Re-initialize database if users have changed + block: + - name: Stop Guacamole containers + ansible.builtin.command: docker stop guac_app guacd + ignore_errors: true + + - name: Remove Guacamole containers + ansible.builtin.command: docker rm guac_app guacd + ignore_errors: true + + - name: Stop and remove PostgreSQL container + ansible.builtin.command: docker stop guac_db + ignore_errors: true + + - name: Remove PostgreSQL container + ansible.builtin.command: docker rm guac_db + ignore_errors: true + + - name: Backup existing database data + ansible.builtin.command: > + tar -czf /opt/guacamole-db/backup-$(date +%Y%m%d-%H%M%S).tar.gz + -C /opt/guacamole-db data/ + ignore_errors: true + + - name: Remove existing database data + ansible.builtin.command: rm -rf /opt/guacamole-db/data/* + ignore_errors: true + + - name: Start Guacamole PostgreSQL container (re-initialized) + ansible.builtin.command: > + docker run -d --name guac_db --network guac_net + -e POSTGRES_USER=guacamole_db + -e POSTGRES_PASSWORD={{ database_password }} + -e POSTGRES_DB=guacamole_db + -v /opt/guacamole-db/data:/var/lib/postgresql/data + -v /opt/guacamole-db/initdb:/docker-entrypoint-initdb.d + --restart always + postgres:latest + register: postgres_restart + failed_when: + - postgres_restart.rc != 0 + - "'already in use' not in postgres_restart.stderr" + + - name: Wait for PostgreSQL to be ready after re-initialization + ansible.builtin.command: docker exec guac_db pg_isready -U guacamole_db + register: postgres_ready_after_restart + until: postgres_ready_after_restart.rc == 0 + retries: 30 + delay: 2 + + when: users_changed + - name: Start guacd container ansible.builtin.command: > docker run -d --name guacd --network guac_net diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml index 4257eb3b52..6099616b86 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml @@ -13,14 +13,54 @@ # limitations under the License. --- +# Check if this role should run +- name: Set current role for lock manager check + ansible.builtin.set_fact: + current_role: "vdi_tool" + +- name: Check if vdi_tool role should run + ansible.builtin.import_role: + name: lock_manager + tasks_from: check_lock + register: lock_check_result + +- name: Set role_should_run from lock check result + ansible.builtin.set_fact: + role_should_run: "{{ lock_check_result.ansible_facts.role_should_run | default(true) }}" + +# Skip all tasks if role should not run +- name: Skip vdi_tool tasks if role should not run + ansible.builtin.debug: + msg: "Skipping vdi_tool role - already completed or not needed" + when: not role_should_run + - name: "Install Guacamole" include_tasks: guacamole.yaml - when: vdi_tool | lower == 'guacamole' + when: + - vdi_tool | lower == 'guacamole' + - role_should_run - name: "Install NoMachine" include_tasks: nomachine.yaml - when: vdi_tool | lower == 'nomachine' + when: + - vdi_tool | lower == 'nomachine' + - role_should_run - name: "Install Workspot" include_tasks: workspot.yaml - when: vdi_tool | lower == 'workspot' + when: + - vdi_tool | lower == 'workspot' + - role_should_run + +# Mark role as completed +- name: Set current role for lock manager completion + ansible.builtin.set_fact: + current_role: "vdi_tool" + role_completed: true + when: role_should_run + +- name: Mark vdi_tool role as completed + ansible.builtin.import_role: + name: lock_manager + tasks_from: create_lock + when: role_should_run diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/connection_bootstrap.sql.j2 b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/connection_bootstrap.sql.j2 deleted file mode 100644 index e90562fccc..0000000000 --- a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/connection_bootstrap.sql.j2 +++ /dev/null @@ -1,72 +0,0 @@ --- VNC Connection for {{ username }} -WITH - user_entity AS ( - SELECT entity_id - FROM guacamole_entity - WHERE name = '{{ username }}' AND type = 'USER' - ), - new_connection AS ( - INSERT INTO guacamole_connection (connection_name, protocol) - VALUES ('{{ username }}', 'vnc') - RETURNING connection_id - ), - insert_params AS ( - INSERT INTO guacamole_connection_parameter (connection_id, parameter_name, parameter_value) - SELECT connection_id, param_name, param_value - FROM new_connection, (VALUES - ('hostname', '{{ ansible_default_ipv4.address }}'), - ('port', '{{ port }}'), - ('username', '{{ username }}'), - ('password', '{{ vnc_password }}') - ) AS params(param_name, param_value) - ), - insert_permissions AS ( - INSERT INTO guacamole_connection_permission (entity_id, connection_id, permission) - SELECT - user_entity.entity_id, - new_connection.connection_id, - perms.permission::guacamole_object_permission_type - FROM user_entity, new_connection, - (VALUES ('READ'),('UPDATE'),('DELETE'),('ADMINISTER')) AS perms(permission) - ) -SELECT 1; - --- SSH Connection for {{ username }} -WITH - user_entity AS ( - SELECT entity_id - FROM guacamole_entity - WHERE name = '{{ username }}' AND type = 'USER' - ), - new_connection_ssh AS ( - INSERT INTO guacamole_connection (connection_name, protocol) - VALUES ('{{ username }} SSH', 'ssh') - RETURNING connection_id - ), - insert_params_ssh AS ( - INSERT INTO guacamole_connection_parameter (connection_id, parameter_name, parameter_value) - SELECT connection_id, param_name, param_value - FROM new_connection_ssh, (VALUES - ('hostname', '{{ ansible_default_ipv4.address }}'), - ('port', '22'), - ('username', '{{ username }}'), - ( - 'private-key', - E'{{ lookup("file", "/home/" ~ username ~ "/.ssh/id_rsa") - | replace("\\", "\\\\") - | replace("'", "''") - | replace("\r", "") - | replace("\n", "\\n") }}' - ) - ) AS params(param_name, param_value) - ), - insert_permissions_ssh AS ( - INSERT INTO guacamole_connection_permission (entity_id, connection_id, permission) - SELECT - user_entity.entity_id, - new_connection_ssh.connection_id, - perms.permission::guacamole_object_permission_type - FROM user_entity, new_connection_ssh, - (VALUES ('READ'),('UPDATE'),('DELETE'),('ADMINISTER')) AS perms(permission) - ) -SELECT 1; diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_bootstrap.sql.j2 b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_bootstrap.sql.j2 new file mode 100644 index 0000000000..cc5e261d23 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_bootstrap.sql.j2 @@ -0,0 +1,78 @@ +-- VNC Connection for {{ username }} (upsert pattern) +-- First, ensure the connection exists +INSERT INTO guacamole_connection (connection_name, protocol) +VALUES ('{{ username }}', 'vnc') +ON CONFLICT (connection_name) DO UPDATE SET + protocol = EXCLUDED.protocol; + +-- Update connection parameters (delete old ones first, then insert new ones) +DELETE FROM guacamole_connection_parameter +WHERE connection_id = (SELECT connection_id FROM guacamole_connection WHERE connection_name = '{{ username }}'); + +INSERT INTO guacamole_connection_parameter (connection_id, parameter_name, parameter_value) +SELECT + connection_id, + param_name, + param_value +FROM guacamole_connection, (VALUES + ('hostname', '{{ ansible_default_ipv4.address }}'), + ('port', '{{ port }}'), + ('username', '{{ username }}'), + ('password', '{{ vnc_password }}') +) AS params(param_name, param_value) +WHERE connection_name = '{{ username }}'; + +-- Update connection permissions (delete old ones first, then insert new ones) +DELETE FROM guacamole_connection_permission +WHERE connection_id = (SELECT connection_id FROM guacamole_connection WHERE connection_name = '{{ username }}') +AND entity_id = (SELECT entity_id FROM guacamole_entity WHERE name = '{{ username }}' AND type = 'USER'); + +INSERT INTO guacamole_connection_permission (entity_id, connection_id, permission) +SELECT + (SELECT entity_id FROM guacamole_entity WHERE name = '{{ username }}' AND type = 'USER'), + (SELECT connection_id FROM guacamole_connection WHERE connection_name = '{{ username }}'), + perms.permission::guacamole_object_permission_type +FROM (VALUES ('READ'),('UPDATE'),('DELETE'),('ADMINISTER')) AS perms(permission); + +-- SSH Connection for {{ username }} (upsert pattern) +-- First, ensure the SSH connection exists +INSERT INTO guacamole_connection (connection_name, protocol) +VALUES ('{{ username }} SSH', 'ssh') +ON CONFLICT (connection_name) DO UPDATE SET + protocol = EXCLUDED.protocol; + +-- Update SSH connection parameters (delete old ones first, then insert new ones) +DELETE FROM guacamole_connection_parameter +WHERE connection_id = (SELECT connection_id FROM guacamole_connection WHERE connection_name = '{{ username }} SSH'); + +INSERT INTO guacamole_connection_parameter (connection_id, parameter_name, parameter_value) +SELECT + connection_id, + param_name, + param_value +FROM guacamole_connection, (VALUES + ('hostname', '{{ ansible_default_ipv4.address }}'), + ('port', '22'), + ('username', '{{ username }}'), + ( + 'private-key', + E'{{ lookup("file", "/home/" ~ username ~ "/.ssh/id_rsa") + | replace("\\", "\\\\") + | replace("'", "''") + | replace("\r", "") + | replace("\n", "\\n") }}' + ) +) AS params(param_name, param_value) +WHERE connection_name = '{{ username }} SSH'; + +-- Update SSH connection permissions (delete old ones first, then insert new ones) +DELETE FROM guacamole_connection_permission +WHERE connection_id = (SELECT connection_id FROM guacamole_connection WHERE connection_name = '{{ username }} SSH') +AND entity_id = (SELECT entity_id FROM guacamole_entity WHERE name = '{{ username }}' AND type = 'USER'); + +INSERT INTO guacamole_connection_permission (entity_id, connection_id, permission) +SELECT + (SELECT entity_id FROM guacamole_entity WHERE name = '{{ username }}' AND type = 'USER'), + (SELECT connection_id FROM guacamole_connection WHERE connection_name = '{{ username }} SSH'), + perms.permission::guacamole_object_permission_type +FROM (VALUES ('READ'),('UPDATE'),('DELETE'),('ADMINISTER')) AS perms(permission); diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/user_bootstrap.sql.j2 b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/user_bootstrap.sql.j2 new file mode 100644 index 0000000000..33d3ef7dee --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/user_bootstrap.sql.j2 @@ -0,0 +1,29 @@ +-- Create or update Guacamole user for {{ username }} +-- First, ensure the entity exists +INSERT INTO guacamole_entity (name, type) +VALUES ('{{ username }}', 'USER') +ON CONFLICT (name, type) DO NOTHING; + +-- Then, insert or update the user with the latest password +INSERT INTO guacamole_user ( + entity_id, + password_salt, + password_hash, + password_date, + disabled, + expired +) +SELECT + entity_id, + NULL, + digest('{{ vdi_password }}', 'SHA-256'), + now(), + false, + false +FROM guacamole_entity +WHERE name = '{{ username }}' AND type = 'USER' +ON CONFLICT (entity_id) DO UPDATE SET + password_hash = EXCLUDED.password_hash, + password_date = EXCLUDED.password_date, + disabled = EXCLUDED.disabled, + expired = EXCLUDED.expired; diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/user_cleanup.sql.j2 b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/user_cleanup.sql.j2 new file mode 100644 index 0000000000..cf323b01dc --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/user_cleanup.sql.j2 @@ -0,0 +1,143 @@ +-- Cleanup script to remove users and connections that are no longer in the configuration +-- This script should be run after all current users have been processed + +{% if vdi_users_updated | length > 0 %} +-- Remove connections for users that no longer exist in the configuration +DELETE FROM guacamole_connection_permission +WHERE connection_id IN ( + SELECT connection_id + FROM guacamole_connection + WHERE connection_name IN ( + -- Remove VNC connections for users not in current config + SELECT connection_name + FROM guacamole_connection + WHERE protocol = 'vnc' + AND connection_name NOT IN ( + {% for user in vdi_users_updated %} + '{{ user.username }}'{% if not loop.last %},{% endif %} + {% endfor %} + ) + ) + OR connection_name IN ( + -- Remove SSH connections for users not in current config + SELECT connection_name + FROM guacamole_connection + WHERE protocol = 'ssh' + AND connection_name NOT IN ( + {% for user in vdi_users_updated %} + '{{ user.username }} SSH'{% if not loop.last %},{% endif %} + {% endfor %} + ) + ) +); + +-- Remove connection parameters for connections that will be deleted +DELETE FROM guacamole_connection_parameter +WHERE connection_id IN ( + SELECT connection_id + FROM guacamole_connection + WHERE connection_name IN ( + -- Remove VNC connections for users not in current config + SELECT connection_name + FROM guacamole_connection + WHERE protocol = 'vnc' + AND connection_name NOT IN ( + {% for user in vdi_users_updated %} + '{{ user.username }}'{% if not loop.last %},{% endif %} + {% endfor %} + ) + ) + OR connection_name IN ( + -- Remove SSH connections for users not in current config + SELECT connection_name + FROM guacamole_connection + WHERE protocol = 'ssh' + AND connection_name NOT IN ( + {% for user in vdi_users_updated %} + '{{ user.username }} SSH'{% if not loop.last %},{% endif %} + {% endfor %} + ) + ) +); + +-- Remove the connections themselves +DELETE FROM guacamole_connection +WHERE connection_name IN ( + -- Remove VNC connections for users not in current config + SELECT connection_name + FROM guacamole_connection + WHERE protocol = 'vnc' + AND connection_name NOT IN ( + {% for user in vdi_users_updated %} + '{{ user.username }}'{% if not loop.last %},{% endif %} + {% endfor %} + ) +) +OR connection_name IN ( + -- Remove SSH connections for users not in current config + SELECT connection_name + FROM guacamole_connection + WHERE protocol = 'ssh' + AND connection_name NOT IN ( + {% for user in vdi_users_updated %} + '{{ user.username }} SSH'{% if not loop.last %},{% endif %} + {% endfor %} + ) +); + +-- Remove user permissions for users that no longer exist +DELETE FROM guacamole_connection_permission +WHERE entity_id IN ( + SELECT entity_id + FROM guacamole_entity + WHERE type = 'USER' + AND name NOT IN ( + {% for user in vdi_users_updated %} + '{{ user.username }}'{% if not loop.last %},{% endif %} + {% endfor %} + ) +); + +-- Remove the users themselves +DELETE FROM guacamole_user +WHERE entity_id IN ( + SELECT entity_id + FROM guacamole_entity + WHERE type = 'USER' + AND name NOT IN ( + {% for user in vdi_users_updated %} + '{{ user.username }}'{% if not loop.last %},{% endif %} + {% endfor %} + ) +); + +-- Remove the entities for users that no longer exist +DELETE FROM guacamole_entity +WHERE type = 'USER' +AND name NOT IN ( + {% for user in vdi_users_updated %} + '{{ user.username }}'{% if not loop.last %},{% endif %} + {% endfor %} +); +{% else %} +-- No users in current configuration, remove all non-admin users +DELETE FROM guacamole_connection_permission +WHERE entity_id IN ( + SELECT entity_id + FROM guacamole_entity + WHERE type = 'USER' + AND name != 'guacadmin' +); + +DELETE FROM guacamole_user +WHERE entity_id IN ( + SELECT entity_id + FROM guacamole_entity + WHERE type = 'USER' + AND name != 'guacadmin' +); + +DELETE FROM guacamole_entity +WHERE type = 'USER' +AND name != 'guacadmin'; +{% endif %} \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/user_bootstrap.sql.j2 b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/user_bootstrap.sql.j2 deleted file mode 100644 index fb6fc25571..0000000000 --- a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/user_bootstrap.sql.j2 +++ /dev/null @@ -1,22 +0,0 @@ --- Create Guacamole entity + user for {{ username }} -WITH new_entity AS ( - INSERT INTO guacamole_entity (name, type) - VALUES ('{{ username }}', 'USER') - RETURNING entity_id -) -INSERT INTO guacamole_user ( - entity_id, - password_salt, - password_hash, - password_date, - disabled, - expired -) -SELECT - entity_id, - NULL, - digest('{{ vdi_password }}', 'SHA-256'), - now(), - false, - false -FROM new_entity; diff --git a/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml index 1c1eadd0e3..3507180468 100644 --- a/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml @@ -13,10 +13,48 @@ # limitations under the License. --- +# Check if this role should run +- name: Set current role for lock manager check + ansible.builtin.set_fact: + current_role: "vnc" + +- name: Check if vnc role should run + ansible.builtin.import_role: + name: lock_manager + tasks_from: check_lock + register: lock_check_result + +- name: Set role_should_run from lock check result + ansible.builtin.set_fact: + role_should_run: "{{ lock_check_result.ansible_facts.role_should_run | default(true) }}" + +# Skip all tasks if role should not run +- name: Skip vnc tasks if role should not run + ansible.builtin.debug: + msg: "Skipping vnc role - already completed or not needed" + when: not role_should_run + - name: "Install TigerVNC" include_tasks: tigervnc.yaml - when: vnc_flavor | lower == 'tigervnc' + when: + - vnc_flavor | lower == 'tigervnc' + - role_should_run - name: "Install TightVNC" include_tasks: tightvnc.yaml - when: vnc_flavor | lower == 'tightvnc' + when: + - vnc_flavor | lower == 'tightvnc' + - role_should_run + +# Mark role as completed +- name: Set current role for lock manager completion + ansible.builtin.set_fact: + current_role: "vnc" + role_completed: true + when: role_should_run + +- name: Mark vnc role as completed + ansible.builtin.import_role: + name: lock_manager + tasks_from: create_lock + when: role_should_run diff --git a/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl b/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl index eed3983fb0..ac1de84902 100644 --- a/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl +++ b/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl @@ -6,7 +6,14 @@ - "/tmp/vdi/vars.yaml" tasks: + # Initialize lock manager + - name: Initialize lock manager + ansible.builtin.import_role: + name: lock_manager + tasks_from: check_lock + %{ for r in roles } - - import_role: + - name: Execute ${r} role + ansible.builtin.import_role: name: "${r}" %{ endfor } diff --git a/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl b/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl index 9960ecf9db..25e202546d 100644 --- a/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl +++ b/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl @@ -8,6 +8,8 @@ vdi_tool: ${vdi_tool} vdi_user_group: ${vdi_user_group} vdi_webapp_port: ${vdi_webapp_port} vdi_resolution: ${vdi_resolution} +vdi_bucket_name: ${vdi_bucket_name} +zone: ${zone} vdi_users: %{ for user in vdi_users } diff --git a/community/modules/scripts/vdi-setup/variables.tf b/community/modules/scripts/vdi-setup/variables.tf index 38cf7bd42a..1dd7ffd75e 100644 --- a/community/modules/scripts/vdi-setup/variables.tf +++ b/community/modules/scripts/vdi-setup/variables.tf @@ -27,6 +27,11 @@ variable "region" { type = string } +variable "zone" { + description = "Zone in which the VDI instances are created." + type = string +} + variable "labels" { description = "Key-value pairs of labels to be added to created resources." type = map(string) @@ -89,3 +94,9 @@ variable "vnc_port_max" { default = 5999 description = "Maximum valid VNC port." } + +variable "debug" { + type = bool + default = false + description = "Enable debug mode for verbose logging during VDI setup." +} From 579de9e204b07442b41f2c4f97e5a9e2775d041c Mon Sep 17 00:00:00 2001 From: Scott Gordon Date: Fri, 1 Aug 2025 12:59:30 +0100 Subject: [PATCH 17/21] vdi module - password reset handling --- community/modules/scripts/vdi-setup/README.md | 120 +- .../VDI_IDEMPOTENCY_IMPLEMENTATION_PLAN.md | 175 - community/modules/scripts/vdi-setup/logs.txt | 14395 ---------------- community/modules/scripts/vdi-setup/main.tf | 59 +- .../roles/lock_manager/tasks/create_lock.yaml | 31 +- .../roles/lock_manager/tasks/main.yaml | 1 + .../lock_manager/tasks/monitor_setup.yaml | 73 - .../lock_manager/templates/lock_file.yaml.j2 | 7 +- .../lock_manager/templates/vdi-monitor.sh.j2 | 184 - .../roles/secret_manager/tasks/main.yaml | 213 +- .../tasks/user_secret_tasks.yaml | 60 +- .../roles/user_provision/tasks/main.yaml | 1 + .../vdi-setup/roles/vdi_monitor/README.md | 200 + .../roles/vdi_monitor/defaults/main.yaml | 25 + .../roles/vdi_monitor/tasks/main.yaml | 161 + .../templates/vdi-monitor-config.sh.j2 | 138 + .../templates/vdi-monitor-detector.sh.j2 | 341 + .../templates/vdi-monitor-file.sh.j2 | 183 + .../templates/vdi-monitor-test.sh.j2 | 141 + .../templates/vdi-monitor.service.j2 | 0 .../vdi_monitor/templates/vdi-monitor.sh.j2 | 132 + .../roles/vdi_tool/tasks/guacamole.yaml | 496 +- .../vdi-setup/roles/vdi_tool/tasks/main.yaml | 18 + .../guacamole/connection_bootstrap.sql.j2 | 32 +- .../templates/guacamole/user_bootstrap.sql.j2 | 14 +- .../templates/guacamole/user_cleanup.sql.j2 | 143 - .../vdi-setup/roles/vnc/tasks/main.yaml | 18 + .../vdi-setup/templates/install.yaml.tftpl | 8 +- .../vdi-setup/templates/vars.yaml.tftpl | 27 +- .../modules/scripts/vdi-setup/validation.tf | 9 + .../modules/scripts/vdi-setup/variables.tf | 15 +- 31 files changed, 2053 insertions(+), 15367 deletions(-) delete mode 100644 community/modules/scripts/vdi-setup/VDI_IDEMPOTENCY_IMPLEMENTATION_PLAN.md delete mode 100644 community/modules/scripts/vdi-setup/logs.txt delete mode 100644 community/modules/scripts/vdi-setup/roles/lock_manager/tasks/monitor_setup.yaml delete mode 100644 community/modules/scripts/vdi-setup/roles/lock_manager/templates/vdi-monitor.sh.j2 create mode 100644 community/modules/scripts/vdi-setup/roles/vdi_monitor/README.md create mode 100644 community/modules/scripts/vdi-setup/roles/vdi_monitor/defaults/main.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/vdi_monitor/tasks/main.yaml create mode 100644 community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-config.sh.j2 create mode 100644 community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-detector.sh.j2 create mode 100644 community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-file.sh.j2 create mode 100644 community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-test.sh.j2 rename community/modules/scripts/vdi-setup/roles/{lock_manager => vdi_monitor}/templates/vdi-monitor.service.j2 (100%) create mode 100644 community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor.sh.j2 delete mode 100644 community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/user_cleanup.sql.j2 diff --git a/community/modules/scripts/vdi-setup/README.md b/community/modules/scripts/vdi-setup/README.md index 3cf4feacf0..928a278a43 100644 --- a/community/modules/scripts/vdi-setup/README.md +++ b/community/modules/scripts/vdi-setup/README.md @@ -2,16 +2,37 @@ Creates a containerised Guacamole instance. Works with the Rocky, Debian and Ubuntu images shown in blueprint example below. -### Secret Manager Integration +## Features -The VDI module supports flexible Secret Manager integration: +- **VDI Tool Support**: Currently supports Guacamole for web-based VDI access +- **User Provisioning**: Supports local user creation with secure password management +- **VNC Integration**: Configures VNC servers for desktop access +- **Secret Manager Integration**: Secure password storage and retrieval for VDI users and webapp admin +- **VDI Monitoring System**: Automatic monitoring and reconfiguration capabilities with password reset flags and deployment status updates +- **Debug Mode**: Comprehensive logging when enabled via the `debug` variable -- **Default Behavior**: Secrets are stored in the deployment project -- **Automatic Password Generation**: If no `secret_name` is provided, random passwords are generated and stored -- **Existing Secret Retrieval**: Provide `secret_name` to use existing secrets from Secret Manager -- **Cross-Project Secrets**: Use `secret_project` to specify a different GCP project if providing secrets +## Secret Manager Integration -## Basic Example (Guacamole) +The module integrates with Google Cloud Secret Manager for secure password handling: + +- **VDI Users**: Passwords are stored in Secret Manager with the pattern `vdi-user-password-{username}-{deployment_name}` +- **Webapp Admin**: Password is stored in Secret Manager with the pattern `webapp-server-password-{deployment_name}` (always stored, not just when reset is enabled) +- **Password Sources**: Users can specify `secret_name` to fetch existing passwords, provide `password` directly, or let the system generate random passwords +- **Reset Functionality**: Both individual user passwords (`reset_password`) and webapp admin password (`reset_webapp_admin_password`) can be forced to regenerate +- **Database Updates**: Password resets update existing database records without re-initializing the entire database + +## VDI Monitoring System + +The module includes a monitoring system that: + +- **Deployment Status**: Updates instance metadata to reflect deployment state (`available`, `reconfiguring`) +- **Password Reset Flags**: Supports `reset_password` for individual users and `reset_webapp_admin_password` for the webapp admin account +- **Automatic Reconfiguration**: Detects changes and triggers reconfiguration when needed +- **Status Tracking**: Maintains deployment status through instance metadata + +## Usage + +### Basic Configuration ```yaml blueprint_name: vdi-test @@ -51,9 +72,13 @@ deployment_groups: settings: vnc_flavor: tigervnc vdi_tool: guacamole - user_provision: local_users vdi_user_group: vdiusers vdi_resolution: 1920x1080 + # Enable debug mode for verbose logging + debug: true + # Force reset of webapp admin password (optional) + reset_webapp_admin_password: false + user_provision: local_users vdi_users: # Alice: password generated and saved to Secret Manager in deployment project - username: alice @@ -67,6 +92,10 @@ deployment_groups: port: 5903 secret_name: charlie-password secret_project: another-project-id + # David: auto-generated password with reset flag (triggers password regeneration) + - username: david + port: 5904 + reset_password: true - id: guac_vm source: modules/compute/vm-instance @@ -79,6 +108,8 @@ deployment_groups: #project: debian-cloud #family: ubuntu-2204-lts #project: ubuntu-os-cloud + name_prefix: debian + add_deployment_name_before_prefix: true machine_type: e2-highcpu-8 tags: ["guacamole"] use: @@ -98,18 +129,18 @@ echo -n "BobPassword123" | gcloud secrets create a-password-for-bob --data-file= echo -n "CharliePassword123" | gcloud secrets create charlie-password --data-file=- --project=another-project-id ``` -### Accessing Guacamole VDI +### Accessing VDI After deployment, you can access the VDI in several ways: -1. **Guacamole Web Interface**: +1. **Web Interface** (for Guacamole): - Access web interface: - http://$VM_PUBLIC_IP:8080/guacamole/#/ - - Note: It is not advisable to serve Guacamole directly from a public IP in production environments. You should consider placing the VDI behind a reverse proxy, load balancer, or tunnel to it directly over IAP (see below). + - Note: It is not advisable to serve the web interface directly from a public IP in production environments. You should consider placing the VDI behind a reverse proxy, load balancer, or tunnel to it directly over IAP (see below). - Admin credentials: - - Username: `guacadmin` - - Password: Retrieve the `webapp-server...` secret from Secret Manager + - Username: `guacadmin` (for Guacamole) + - Password: Retrieve the `webapp-server-password-{deployment_name}` secret from Secret Manager 2. **User VDI Access**: - Each user's credentials are stored in Secret Manager @@ -124,65 +155,4 @@ After deployment, you can access the VDI in several ways: --zone=us-central1-a ``` - - Guacamole will then be accessible from http://localhost:8080/guacamole/ - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0 | -| [archive](#requirement\_archive) | ~> 2.0 | -| [google](#requirement\_google) | >= 3.83 | - -## Providers - -| Name | Version | -|------|---------| -| [archive](#provider\_archive) | ~> 2.0 | -| [google](#provider\_google) | >= 3.83 | -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | -| [terraform_data.input_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [archive_file.roles_tar](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | n/a | yes | -| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | -| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | -| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | -| [user\_provision](#input\_user\_provision) | User type to create (local\_users supported. os-login to do. | `string` | `"local_users"` | no | -| [vdi\_resolution](#input\_vdi\_resolution) | Desktop resolution for VNC sessions (e.g. 1920x1080). | `string` | `"1920x1080"` | no | -| [vdi\_tool](#input\_vdi\_tool) | VDI tool to deploy (guacamole currently supported). | `string` | `"guacamole"` | no | -| [vdi\_user\_group](#input\_vdi\_user\_group) | Unix group to create/use for VDI users. | `string` | `"vdiusers"` | no | -| [vdi\_users](#input\_vdi\_users) | List of VDI users to configure. Passwords are handled securely by the Ansible roles: if secret\_name is provided, the password is fetched from Secret Manager; if neither password nor secret\_name is provided, a random password is generated and stored in Secret Manager. If secret\_project is provided, it specifies the GCP project where the secret is stored (defaults to the deployment project). | 
list(object({
    username       = string
    port           = number
    secret_name    = optional(string)
    secret_project = optional(string)
  }))
| `[]` | no | -| [vdi\_webapp\_port](#input\_vdi\_webapp\_port) | Port to serve the Webapp interface from if applicable | `string` | `"8080"` | no | -| [vnc\_flavor](#input\_vnc\_flavor) | The VNC server flavor to use (tigervnc currently supported) | `string` | `"tigervnc"` | no | -| [vnc\_port\_max](#input\_vnc\_port\_max) | Maximum valid VNC port. | `number` | `5999` | no | -| [vnc\_port\_min](#input\_vnc\_port\_min) | Minimum valid VNC port. | `number` | `5901` | no | -| [debug](#input\_debug) | Enable debug mode for verbose logging during VDI setup. | `bool` | `false` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [guacamole\_admin\_password\_secret](#output\_guacamole\_admin\_password\_secret) | The name of the Secret Manager secret containing the Guacamole admin password | -| [guacamole\_admin\_username](#output\_guacamole\_admin\_username) | The admin username for Guacamole | -| [startup\_script](#output\_startup\_script) | Combined startup script that installs VDI (VNC, Guacamole, users). | -| [vdi\_runner](#output\_vdi\_runner) | Shell runner wrapping Ansible playbook + roles (for custom-image or direct use). | -| [vdi\_user\_credentials](#output\_vdi\_user\_credentials) | Map of VDI user credentials stored in Secret Manager | - + - The web interface will then be accessible from http://localhost:8080/guacamole/ (for Guacamole) diff --git a/community/modules/scripts/vdi-setup/VDI_IDEMPOTENCY_IMPLEMENTATION_PLAN.md b/community/modules/scripts/vdi-setup/VDI_IDEMPOTENCY_IMPLEMENTATION_PLAN.md deleted file mode 100644 index 4f1faf9eed..0000000000 --- a/community/modules/scripts/vdi-setup/VDI_IDEMPOTENCY_IMPLEMENTATION_PLAN.md +++ /dev/null @@ -1,175 +0,0 @@ -# VDI Idempotency Implementation Plan - -## Overview -This document outlines the implementation of idempotency for the VDI setup module, ensuring that roles can be safely re-run without causing conflicts or duplicate operations. - -## Key Features - -### 1. Lock File System -- **Location**: `/opt/vdi-setup/.vdi-lock.yaml` -- **Purpose**: Tracks deployment status, role completion, and configuration changes -- **Structure**: YAML format with deployment metadata and role status - -### 2. Role Execution Control -- **Pattern**: Each role checks if it should run before executing tasks -- **Logic**: Roles run if: - - Lock file doesn't exist (fresh deployment) - - Role is not marked as completed - - Deployment configuration has changed - - Force rerun is enabled - -### 3. Setup Status Tracking -- **Status Values**: `configuring`, `available`, `error` -- **Purpose**: Indicates current VDI setup state for external monitoring -- **Usage**: Django app can show "Offline" if VDI is offline (separate logic) - -### 4. VM Metadata Integration -- **Content**: Base64-encoded lock file stored in VM metadata -- **Purpose**: External access to VDI status without SSH access -- **Command**: `gcloud compute instances describe --format="value(metadata.items[vdi-lock-content])" | base64 -d` - -### 5. Monitoring Service -- **Service**: `vdi-monitor.service` (systemd) -- **Script**: `/opt/vdi-setup/vdi-monitor.sh` -- **Purpose**: Monitors lock file changes and triggers reconfiguration -- **Bucket Sync**: Pulls latest files from GCS bucket before reconfiguring - -## Implementation Details - -### Lock File Structure -```yaml -vdi_setup_status: - deployment_name: "vdi-test-scott" - deployment_hash: "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab" - lock_version: "1.0" - created_at: "2025-07-24T10:13:31Z" - last_updated: "2025-07-24T10:13:31Z" - force_rerun: false - setup_status: "configuring" # configuring, available, error - - completed_roles: - base_os: - completed: false - lock_manager: - completed: true - completed_at: "2025-07-24T10:13:31Z" - secret_manager: - completed: false - user_provision: - completed: false - vnc: - completed: false - vdi_tool: - completed: false - - user_secrets_status: - last_secret_check: "2025-07-24T10:13:31Z" - user_secrets_hash: "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" - users_updated: - - alice - - bob - - user_management: - current_users: - - alice - - bob - previous_users: - - alice - - bob - removed_users: [] -``` - -### Role Execution Pattern -Each role follows this pattern: -```yaml -# Check if this role should run -- name: Check if role should run - ansible.builtin.import_role: - name: lock_manager - tasks_from: check_lock - vars: - current_role: "" - -# Skip all tasks if role should not run -- name: Skip tasks if role should not run - ansible.builtin.debug: - msg: "Skipping role - already completed or not needed" - when: not role_should_run - -# Role-specific tasks with conditional execution -- name: Role task - ansible.builtin.task: - # task details - when: role_should_run - -# Mark role as completed -- name: Mark role as completed - ansible.builtin.import_role: - name: lock_manager - tasks_from: create_lock - vars: - current_role: "" - role_completed: true - when: role_should_run -``` - -### Change Detection -- **Deployment Hash**: SHA-256 hash of deployment configuration -- **User Secrets Hash**: SHA-256 hash of user configuration (blueprint data only) -- **Role Completion**: Individual tracking of each role's completion status - -### Secret Manager Integration -- **Blueprint-based Detection**: User secrets hash is calculated from blueprint data (usernames, ports, secret_names, etc.) -- **Password Changes**: Changes to passwords in Secret Manager are **not automatically detected** -- **Manual Re-initialization**: To apply password changes, use `force_rerun: true` in lock file -- **Secret ID Changes**: Changes to `secret_name` references are automatically detected - -### Monitoring Service -- **Trigger**: Lock file modification detected -- **Action**: Sync files from GCS bucket and run Ansible reconfiguration -- **Logging**: Detailed logs in `/var/log/vdi-monitor.log` - -## Benefits - -1. **Idempotency**: Safe to re-run deployment without conflicts -2. **Efficiency**: Skip completed roles to reduce deployment time -3. **Monitoring**: External visibility into VDI status -4. **Automation**: Automatic reconfiguration on configuration changes -5. **Reliability**: Proper error handling and status tracking - -## Usage - -### Manual Status Check -```bash -# Check on-disk lock file -cat /opt/vdi-setup/.vdi-lock.yaml - -# Check VM metadata -gcloud compute instances describe --zone= --format="value(metadata.items[vdi-lock-content])" | base64 -d -``` - -### Force Re-run -```bash -# Edit lock file to force re-run -sed -i 's/force_rerun: false/force_rerun: true/' /opt/vdi-setup/.vdi-lock.yaml -``` - -### Handle Secret Manager Password Changes -```bash -# To apply password changes from Secret Manager: -# 1. Update the password in Secret Manager -# 2. Force re-run to apply changes -sed -i 's/force_rerun: false/force_rerun: true/' /opt/vdi-setup/.vdi-lock.yaml - -# 3. Run the playbook again -cd /tmp/vdi && ansible-playbook install.yaml --extra-vars @vars.yaml -``` - -### Monitor Service -```bash -# Check service status -systemctl status vdi-monitor - -# View logs -tail -f /var/log/vdi-monitor.log -``` \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/logs.txt b/community/modules/scripts/vdi-setup/logs.txt deleted file mode 100644 index f5c0e0a2f5..0000000000 --- a/community/modules/scripts/vdi-setup/logs.txt +++ /dev/null @@ -1,14395 +0,0 @@ -[root@vdi-test-scott-rocky-0 ~]# cat /var/log/messages | awk -F'startup-script: ' '/startup-script:/ {print $2}' -77 files removed -+ rm -rf /var/cache/yum -+ shred --remove /etc/ssh/ssh_host_ecdsa_key /etc/ssh/ssh_host_ecdsa_key.pub /etc/ssh/ssh_host_ed25519_key /etc/ssh/ssh_host_ed25519_key.pub /etc/ssh/ssh_host_rsa_key /etc/ssh/ssh_host_rsa_key.pub -+ truncate -s 0 /var/log/messages -+ rm -rf /root/mpi-tuning/ -+ rm -f /root/.bash_history -+ history -c -+ echo 'DaisySuccess: HPC VM tuning complete' -DaisySuccess: HPC VM tuning complete -Thu Jul 24 15:23:27 +0000 2025 Info [1535]: === start executing runner: 99-running-script-warning.sh-54ee === -Thu Jul 24 15:23:27 +0000 2025 Info [1535]: === 99-running-script-warning.sh-54ee finished with exit_code=0 === -Thu Jul 24 15:23:29 +0000 2025 Info [1535]: === start executing runner: early_run_hotfixes.sh-b383 === -Thu Jul 24 15:23:29 +0000 2025 Info [1535]: === early_run_hotfixes.sh-b383 finished with exit_code=0 === -Thu Jul 24 15:23:30 +0000 2025 Info [1535]: === start executing runner: passed_startup_script.sh-22f5 === -Thu Jul 24 15:23:32 +0000 2025 Info [1870]: === start executing runner: 99-running-script-warning.sh-54ee === -Thu Jul 24 15:23:32 +0000 2025 Info [1870]: === 99-running-script-warning.sh-54ee finished with exit_code=0 === -Thu Jul 24 15:23:34 +0000 2025 Info [1870]: === start executing runner: early_run_hotfixes.sh-b383 === -Thu Jul 24 15:23:34 +0000 2025 Info [1870]: === early_run_hotfixes.sh-b383 finished with exit_code=0 === -Thu Jul 24 15:23:36 +0000 2025 Info [1870]: === start executing runner: install_ansible_automatic.sh-ccf1 === -+ REQ_ANSIBLE_VERSION=2.15 -+ REQ_ANSIBLE_PIP_VERSION=8.7.0 -+ REQ_PIP_WHEEL_VERSION=0.45.1 -+ REQ_PIP_SETUPTOOLS_VERSION=80.8.0 -+ REQ_PIP_MAJOR_VERSION=25 -+ REQ_PYTHON3_VERSION=9 -+ main /usr/local/ghpc-venv -+ '[' 1 -gt 1 ']' -+ venv_path=/usr/local/ghpc-venv -+ get_python_path -+ python_path= -+ command -v python3 -++ command -v python3 -+ python_path=/usr/bin/python3 -+ get_python_major_version /usr/bin/python3 -+ python_path=/usr/bin/python3 -++ /usr/bin/python3 -c 'import sys; print(sys.version_info.major)' -+ python_major_version=3 -+ get_python_minor_version /usr/bin/python3 -+ python_path=/usr/bin/python3 -++ /usr/bin/python3 -c 'import sys; print(sys.version_info.minor)' -+ python_minor_version=6 -+ '[' /usr/bin/python3 = '' ']' -+ '[' 3 = 2 ']' -+ '[' 6 -lt 9 ']' -+ install_python3 -+ '[' -f /etc/redhat-release ']' -+ install_python3_dnf -++ rpm -E '%{rhel}' -+ major_version=8 -+ set -- '--disablerepo=*' --enablerepo=baseos,appstream -+ grep -qi 'ID="rhel"' /etc/os-release -+ '[' 8 -lt 10 ']' -+ dnf install '--disablerepo=*' --enablerepo=baseos,appstream -y python3.12 python3.12-pip -Waiting for process with pid 1124 to finish. -Last metadata expiration check: 0:00:46 ago on Thu 24 Jul 2025 03:24:31 PM UTC. -Dependencies resolved. -================================================================================ - Package Arch Version Repository Size -================================================================================ -Installing: - python3.12 x86_64 3.12.11-1.el8_10 appstream 30 k - python3.12-pip noarch 23.2.1-4.el8 appstream 3.2 M -Installing dependencies: - python3.12-libs x86_64 3.12.11-1.el8_10 appstream 10 M - python3.12-pip-wheel noarch 23.2.1-4.el8 appstream 1.5 M -Installing weak dependencies: - python3.12-setuptools noarch 68.2.2-5.el8_10 appstream 1.7 M - -Transaction Summary -================================================================================ -Install 5 Packages - -Total download size: 16 M -Installed size: 65 M -Downloading Packages: -(1/5): python3.12-3.12.11-1.el8_10.x86_64.rpm 858 kB/s | 30 kB 00:00 -(2/5): python3.12-libs-3.12.11-1.el8_10.x86_64. 73 MB/s | 10 MB 00:00 -(3/5): python3.12-pip-wheel-23.2.1-4.el8.noarch 11 MB/s | 1.5 MB 00:00 -(4/5): python3.12-setuptools-68.2.2-5.el8_10.no 6.1 MB/s | 1.7 MB 00:00 -(5/5): python3.12-pip-23.2.1-4.el8.noarch.rpm 6.0 MB/s | 3.2 MB 00:00 --------------------------------------------------------------------------------- -Total 27 MB/s | 16 MB 00:00 -Running transaction check -Transaction check succeeded. -Running transaction test -Transaction test succeeded. -Running transaction - Preparing : 1/1 - Installing : python3.12-pip-wheel-23.2.1-4.el8.noarch 1/5 - Installing : python3.12-libs-3.12.11-1.el8_10.x86_64 2/5 - Installing : python3.12-3.12.11-1.el8_10.x86_64 3/5 - Running scriptlet: python3.12-3.12.11-1.el8_10.x86_64 3/5 - Installing : python3.12-setuptools-68.2.2-5.el8_10.noarch 4/5 - Installing : python3.12-pip-23.2.1-4.el8.noarch 5/5 - Running scriptlet: python3.12-pip-23.2.1-4.el8.noarch 5/5 - Verifying : python3.12-3.12.11-1.el8_10.x86_64 1/5 - Verifying : python3.12-libs-3.12.11-1.el8_10.x86_64 2/5 - Verifying : python3.12-pip-23.2.1-4.el8.noarch 3/5 - Verifying : python3.12-pip-wheel-23.2.1-4.el8.noarch 4/5 - Verifying : python3.12-setuptools-68.2.2-5.el8_10.noarch 5/5 - -Installed: - python3.12-3.12.11-1.el8_10.x86_64 - python3.12-libs-3.12.11-1.el8_10.x86_64 - python3.12-pip-23.2.1-4.el8.noarch - python3.12-pip-wheel-23.2.1-4.el8.noarch - python3.12-setuptools-68.2.2-5.el8_10.noarch - -Complete! -++ command -v python3.12 -+ python_path=/usr/bin/python3.12 -+ get_python_major_version /usr/bin/python3.12 -+ python_path=/usr/bin/python3.12 -++ /usr/bin/python3.12 -c 'import sys; print(sys.version_info.major)' -+ python_major_version=3 -+ get_python_minor_version /usr/bin/python3.12 -+ python_path=/usr/bin/python3.12 -++ /usr/bin/python3.12 -c 'import sys; print(sys.version_info.minor)' -+ python_minor_version=12 -+ /usr/bin/python3.12 -m pip --version -pip 23.2.1 from /usr/lib/python3.12/site-packages/pip (python 3.12) -+ /usr/bin/python3.12 -m venv /usr/local/ghpc-venv --copies -+ venv_python_path=/usr/local/ghpc-venv/bin/python3 -++ /usr/local/ghpc-venv/bin/python3 -m pip --version -++ sed -nr 's/^pip ([0-9]+\.[0-9]+).*$/\1/p' -+ pip_version=23.2 -++ echo 23.2 -++ cut -d . -f 1 -+ pip_major_version=23 -+ '[' 23 -lt 25 ']' -+ /usr/local/ghpc-venv/bin/python3 -m pip install --upgrade pip -Requirement already satisfied: pip in /usr/local/ghpc-venv/lib64/python3.12/site-packages (23.2.1) -Collecting pip - Obtaining dependency information for pip from https://files.pythonhosted.org/packages/29/a2/d40fb2460e883eca5199c62cfc2463fd261f760556ae6290f88488c362c0/pip-25.1.1-py3-none-any.whl.metadata - Downloading pip-25.1.1-py3-none-any.whl.metadata (3.6 kB) -Downloading pip-25.1.1-py3-none-any.whl (1.8 MB) - ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā” 1.8/1.8 MB 28.2 MB/s eta 0:00:00 -Installing collected packages: pip - Attempting uninstall: pip - Found existing installation: pip 23.2.1 - Uninstalling pip-23.2.1: - Successfully uninstalled pip-23.2.1 -Successfully installed pip-25.1.1 -++ /usr/local/ghpc-venv/bin/python3 -m pip list --format=freeze -++ grep '^wheel' -++ true -+ wheel_pkg= -+ '[' '' '!=' wheel==0.45.1 ']' -+ /usr/local/ghpc-venv/bin/python3 -m pip install -U wheel==0.45.1 -Collecting wheel==0.45.1 - Downloading wheel-0.45.1-py3-none-any.whl.metadata (2.3 kB) -Downloading wheel-0.45.1-py3-none-any.whl (72 kB) -Installing collected packages: wheel -Successfully installed wheel-0.45.1 -++ /usr/local/ghpc-venv/bin/python3 -m pip list --format=freeze -++ grep '^setuptools' -++ true -+ setuptools_pkg= -+ '[' '' '!=' setuptools==80.8.0 ']' -+ /usr/local/ghpc-venv/bin/python3 -m pip install -U setuptools==80.8.0 -Collecting setuptools==80.8.0 - Downloading setuptools-80.8.0-py3-none-any.whl.metadata (6.6 kB) -Downloading setuptools-80.8.0-py3-none-any.whl (1.2 MB) - ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā” 1.2/1.2 MB 29.3 MB/s eta 0:00:00 -Installing collected packages: setuptools -Successfully installed setuptools-80.8.0 -+ '[' '!' -f /etc/ansible/ansible.cfg ']' -+ mkdir /etc/ansible -+ cat -+ ansible_version= -+ command -v ansible-playbook -+ '[' -z '' ']' -+ /usr/local/ghpc-venv/bin/python3 -m pip install ansible==8.7.0 -Collecting ansible==8.7.0 - Downloading ansible-8.7.0-py3-none-any.whl.metadata (7.9 kB) -Collecting ansible-core~=2.15.7 (from ansible==8.7.0) - Downloading ansible_core-2.15.13-py3-none-any.whl.metadata (7.0 kB) -Collecting jinja2>=3.0.0 (from ansible-core~=2.15.7->ansible==8.7.0) - Downloading jinja2-3.1.6-py3-none-any.whl.metadata (2.9 kB) -Collecting PyYAML>=5.1 (from ansible-core~=2.15.7->ansible==8.7.0) - Downloading PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (2.1 kB) -Collecting cryptography (from ansible-core~=2.15.7->ansible==8.7.0) - Downloading cryptography-45.0.5-cp311-abi3-manylinux_2_28_x86_64.whl.metadata (5.7 kB) -Collecting packaging (from ansible-core~=2.15.7->ansible==8.7.0) - Downloading packaging-25.0-py3-none-any.whl.metadata (3.3 kB) -Collecting resolvelib<1.1.0,>=0.5.3 (from ansible-core~=2.15.7->ansible==8.7.0) - Downloading resolvelib-1.0.1-py2.py3-none-any.whl.metadata (4.0 kB) -Collecting MarkupSafe>=2.0 (from jinja2>=3.0.0->ansible-core~=2.15.7->ansible==8.7.0) - Downloading MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.0 kB) -Collecting cffi>=1.14 (from cryptography->ansible-core~=2.15.7->ansible==8.7.0) - Downloading cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.5 kB) -Collecting pycparser (from cffi>=1.14->cryptography->ansible-core~=2.15.7->ansible==8.7.0) - Downloading pycparser-2.22-py3-none-any.whl.metadata (943 bytes) -Downloading ansible-8.7.0-py3-none-any.whl (48.4 MB) - ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā” 48.4/48.4 MB 102.4 MB/s eta 0:00:00 -Downloading ansible_core-2.15.13-py3-none-any.whl (2.3 MB) - ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā” 2.3/2.3 MB 105.5 MB/s eta 0:00:00 -Downloading resolvelib-1.0.1-py2.py3-none-any.whl (17 kB) -Downloading jinja2-3.1.6-py3-none-any.whl (134 kB) -Downloading MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (23 kB) -Downloading PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (767 kB) - ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā” 767.5/767.5 kB 59.6 MB/s eta 0:00:00 -Downloading cryptography-45.0.5-cp311-abi3-manylinux_2_28_x86_64.whl (4.5 MB) - ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā” 4.5/4.5 MB 153.6 MB/s eta 0:00:00 -Downloading cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (479 kB) -Downloading packaging-25.0-py3-none-any.whl (66 kB) -Downloading pycparser-2.22-py3-none-any.whl (117 kB) -Installing collected packages: resolvelib, PyYAML, pycparser, packaging, MarkupSafe, jinja2, cffi, cryptography, ansible-core, ansible - -Successfully installed MarkupSafe-3.0.2 PyYAML-6.0.2 ansible-8.7.0 ansible-core-2.15.13 cffi-1.17.1 cryptography-45.0.5 jinja2-3.1.6 packaging-25.0 pycparser-2.22 resolvelib-1.0.1 -+ read -r cmd -+ '[' -L /usr/bin/ansible ']' -+ ln -s /usr/local/ghpc-venv/bin/ansible /usr/bin/ansible -+ read -r cmd -+ '[' -L /usr/bin/ansible-config ']' -+ ln -s /usr/local/ghpc-venv/bin/ansible-config /usr/bin/ansible-config -+ read -r cmd -+ '[' -L /usr/bin/ansible-connection ']' -+ ln -s /usr/local/ghpc-venv/bin/ansible-connection /usr/bin/ansible-connection -+ read -r cmd -+ '[' -L /usr/bin/ansible-console ']' -+ ln -s /usr/local/ghpc-venv/bin/ansible-console /usr/bin/ansible-console -+ read -r cmd -+ '[' -L /usr/bin/ansible-doc ']' -+ ln -s /usr/local/ghpc-venv/bin/ansible-doc /usr/bin/ansible-doc -+ read -r cmd -+ '[' -L /usr/bin/ansible-galaxy ']' -+ ln -s /usr/local/ghpc-venv/bin/ansible-galaxy /usr/bin/ansible-galaxy -+ read -r cmd -+ '[' -L /usr/bin/ansible-inventory ']' -+ ln -s /usr/local/ghpc-venv/bin/ansible-inventory /usr/bin/ansible-inventory -+ read -r cmd -+ '[' -L /usr/bin/ansible-playbook ']' -+ ln -s /usr/local/ghpc-venv/bin/ansible-playbook /usr/bin/ansible-playbook -+ read -r cmd -+ '[' -L /usr/bin/ansible-pull ']' -+ ln -s /usr/local/ghpc-venv/bin/ansible-pull /usr/bin/ansible-pull -+ read -r cmd -+ '[' -L /usr/bin/ansible-test ']' -+ ln -s /usr/local/ghpc-venv/bin/ansible-test /usr/bin/ansible-test -+ read -r cmd -+ '[' -L /usr/bin/ansible-vault ']' -+ ln -s /usr/local/ghpc-venv/bin/ansible-vault /usr/bin/ansible-vault -+ read -r cmd -Thu Jul 24 15:25:55 +0000 2025 Info [1870]: === install_ansible_automatic.sh-ccf1 finished with exit_code=0 === -Thu Jul 24 15:25:57 +0000 2025 Info [1870]: === start executing runner: ansible_docker_settings.json-c21f === -Thu Jul 24 15:25:57 +0000 2025 Info [1870]: === ansible_docker_settings.json-c21f finished with exit_code=0 === -Thu Jul 24 15:25:58 +0000 2025 Info [1870]: === start executing runner: install_docker.yml-b4a8 === - -PLAY [Install and configure Docker] ******************************************** - -TASK [Gathering Facts] ********************************************************* -ok: [localhost] - -TASK [Gather distribution facts] *********************************************** -ok: [localhost] - -TASK [Check if docker is installed] ******************************************** -ok: [localhost] - -TASK [Add Docker GPG key (Rocky)] ********************************************** -changed: [localhost] - -TASK [Add Docker repository for Rocky Linux] *********************************** -changed: [localhost] - -TASK [Install Docker Engine and related packages on Rocky] ********************* -changed: [localhost] - -TASK [Install python3-pip] ***************************************************** -ok: [localhost] - -TASK [Install Docker SDK for Python] ******************************************* -changed: [localhost] - -TASK [Download Docker Installer (get.docker.com)] ****************************** -skipping: [localhost] - -TASK [Install Docker] ********************************************************** -skipping: [localhost] - -TASK [Create Docker daemon configuration] ************************************** -skipping: [localhost] - -TASK [Create Docker service override directory] ******************************** -changed: [localhost] - -TASK [Create Docker service override configuration] **************************** -changed: [localhost] - -TASK [Create Docker socket override directory] ********************************* -skipping: [localhost] - -TASK [Create Docker socket override configuration] ***************************** -skipping: [localhost] - -TASK [Delete Docker socket override configuration] ***************************** -ok: [localhost] - -RUNNING HANDLER [Refresh dnf cache] ******************************************** -ok: [localhost] - -TASK [Start Docker] ************************************************************ -changed: [localhost] - -PLAY RECAP ********************************************************************* -localhost : ok=13 changed=7 unreachable=0 failed=0 skipped=5 rescued=0 ignored=0 - -Thu Jul 24 15:27:29 +0000 2025 Info [1870]: === install_docker.yml-b4a8 finished with exit_code=0 === -Thu Jul 24 15:27:31 +0000 2025 Info [1870]: === start executing runner: install-deps.sh-1526 === -+ /usr/local/ghpc-venv/bin/python3 -m pip install requests google-auth docker -Requirement already satisfied: requests in /usr/local/ghpc-venv/lib64/python3.12/site-packages (2.32.4) -Collecting google-auth - Downloading google_auth-2.40.3-py2.py3-none-any.whl.metadata (6.2 kB) -Requirement already satisfied: docker in /usr/local/ghpc-venv/lib64/python3.12/site-packages (7.1.0) -Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/ghpc-venv/lib64/python3.12/site-packages (from requests) (3.4.2) -Requirement already satisfied: idna<4,>=2.5 in /usr/local/ghpc-venv/lib64/python3.12/site-packages (from requests) (3.10) -Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/ghpc-venv/lib64/python3.12/site-packages (from requests) (2.5.0) -Requirement already satisfied: certifi>=2017.4.17 in /usr/local/ghpc-venv/lib64/python3.12/site-packages (from requests) (2025.7.14) -Collecting cachetools<6.0,>=2.0.0 (from google-auth) - Downloading cachetools-5.5.2-py3-none-any.whl.metadata (5.4 kB) -Collecting pyasn1-modules>=0.2.1 (from google-auth) - Downloading pyasn1_modules-0.4.2-py3-none-any.whl.metadata (3.5 kB) -Collecting rsa<5,>=3.1.4 (from google-auth) - Downloading rsa-4.9.1-py3-none-any.whl.metadata (5.6 kB) -Collecting pyasn1>=0.1.3 (from rsa<5,>=3.1.4->google-auth) - Downloading pyasn1-0.6.1-py3-none-any.whl.metadata (8.4 kB) -Downloading google_auth-2.40.3-py2.py3-none-any.whl (216 kB) -Downloading cachetools-5.5.2-py3-none-any.whl (10 kB) -Downloading rsa-4.9.1-py3-none-any.whl (34 kB) -Downloading pyasn1-0.6.1-py3-none-any.whl (83 kB) -Downloading pyasn1_modules-0.4.2-py3-none-any.whl (181 kB) -Installing collected packages: pyasn1, cachetools, rsa, pyasn1-modules, google-auth - -Successfully installed cachetools-5.5.2 google-auth-2.40.3 pyasn1-0.6.1 pyasn1-modules-0.4.2 rsa-4.9.1 -+ ansible-galaxy collection install google.cloud -Starting galaxy collection install process -Nothing to do. All requested collections are already installed. If you want to reinstall them, consider using `--force`. -Thu Jul 24 15:27:32 +0000 2025 Info [1870]: === install-deps.sh-1526 finished with exit_code=0 === -Thu Jul 24 15:27:34 +0000 2025 Info [1870]: === start executing runner: roles.tar.gz-d489 === -Thu Jul 24 15:27:34 +0000 2025 Info [1870]: === roles.tar.gz-d489 finished with exit_code=0 === -Thu Jul 24 15:27:36 +0000 2025 Info [1870]: === start executing runner: unpack_roles.sh-994e === -+ mkdir -p /tmp/vdi/roles -+ tar xzf /tmp/vdi/roles.tar.gz -C /tmp/vdi/roles -Thu Jul 24 15:27:36 +0000 2025 Info [1870]: === unpack_roles.sh-994e finished with exit_code=0 === -Thu Jul 24 15:27:38 +0000 2025 Info [1870]: === start executing runner: vars.yaml-faf2 === -Thu Jul 24 15:27:38 +0000 2025 Info [1870]: === vars.yaml-faf2 finished with exit_code=0 === -Thu Jul 24 15:27:40 +0000 2025 Info [1870]: === start executing runner: install.yaml-f44d === -Using /etc/ansible/ansible.cfg as config file - -PLAY [localhost] *************************************************************** - -TASK [Gathering Facts] ********************************************************* -ok: [localhost] - -TASK [lock_manager : Check if lock file exists] ******************************** -ok: [localhost] => { - "changed": false, - "stat": { - "exists": false - } -} - -TASK [lock_manager : Debug lock file existence] ******************************** -ok: [localhost] => {} - -MSG: - -Lock file exists: False - -TASK [lock_manager : Load existing lock file] ********************************** -skipping: [localhost] => { - "changed": false, - "false_condition": "lock_file_stat.stat.exists", - "skip_reason": "Conditional result was False" -} - -TASK [lock_manager : Debug lock file load result] ****************************** -skipping: [localhost] => { - "false_condition": "debug | default(false) and lock_file_stat.stat.exists" -} - -TASK [lock_manager : Calculate current deployment hash] ************************ -ok: [localhost] => { - "ansible_facts": { - "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab" - }, - "changed": false -} - -TASK [lock_manager : Debug deployment hash calculation] ************************ -ok: [localhost] => {} - -MSG: - -Current deployment hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab -Deployment components: - - deployment_name: vdi-test-scott - - project_id: hpc-discovery-external - - vdi_tool: guacamole - - vnc_flavor: tigervnc - - user_provision: local_users - - vdi_webapp_port: 8081 - - vdi_resolution: 1920x1080 - - vdi_users count: 2 - - -TASK [lock_manager : Calculate current user secrets hash] ********************** -ok: [localhost] => { - "ansible_facts": { - "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" - }, - "changed": false -} - -TASK [lock_manager : Debug user secrets hash calculation] ********************** -ok: [localhost] => {} - -MSG: - -Current user secrets hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 -User configuration components: - - vdi_users (blueprint): [{"username": "alice", "port": 5901}, {"username": "bob", "port": 5902, "secret_name": "a-password-for-bob"}] - - -TASK [lock_manager : Set default values for missing lock file] ***************** -ok: [localhost] => { - "ansible_facts": { - "vdi_setup_status": { - "completed_roles": { - "base_os": { - "completed": false - }, - "lock_manager": { - "completed": false - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "none", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:27:41Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [], - "previous_users": [], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:27:41Z", - "user_secrets_hash": "none", - "users_updated": [] - } - } - }, - "changed": false -} - -TASK [lock_manager : Debug lock file status] *********************************** -ok: [localhost] => {} - -MSG: - -Lock file status: - - exists: False - - deployment_hash: none - - user_secrets_hash: none - - force_rerun: False - - -TASK [lock_manager : Check if lock file exists] ******************************** -ok: [localhost] => { - "ansible_facts": { - "lock_file_exists": false - }, - "changed": false -} - -TASK [lock_manager : Check if current role is completed] *********************** -ok: [localhost] => { - "ansible_facts": { - "current_role_completed": false - }, - "changed": false -} - -TASK [lock_manager : Check if deployment hash matches] ************************* -ok: [localhost] => { - "ansible_facts": { - "deployment_hash_matches": false - }, - "changed": false -} - -TASK [lock_manager : Check if force rerun is enabled] ************************** -ok: [localhost] => { - "ansible_facts": { - "force_rerun_enabled": false - }, - "changed": false -} - -TASK [lock_manager : Determine if role should run] ***************************** -ok: [localhost] => { - "ansible_facts": { - "role_should_run": true - }, - "changed": false -} - -TASK [lock_manager : Debug role should run calculation] ************************ -ok: [localhost] => {} - -MSG: - -Role should run calculation for lock_manager: - - not lock_file_exists: True - - not current_role_completed: True - - not deployment_hash_matches: True - - force_rerun_enabled: False - - role_should_run: True - - -TASK [lock_manager : Debug role execution decision] **************************** -ok: [localhost] => {} - -MSG: - -Role execution decision for lock_manager: - - current_role: not set - - ansible_role_name: lock_manager - - role_should_run: True - - lock_file_exists: False - - current_role_completed: False - - deployment_hash_matches: False - - force_rerun_enabled: False - - -TASK [lock_manager : Check if user configuration has changed] ****************** -ok: [localhost] => { - "ansible_facts": { - "user_config_changed": true - }, - "changed": false -} - -TASK [lock_manager : Set user secrets changed flag (will be updated by secret_manager role)] *** -ok: [localhost] => { - "ansible_facts": { - "user_secrets_changed": true - }, - "changed": false -} - -TASK [lock_manager : Debug user secrets change detection] ********************** -ok: [localhost] => {} - -MSG: - -User configuration change detection: - - user_config_changed: True - - user_secrets_changed: True - - stored_hash: none - - current_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 - - -TASK [lock_manager : Set lock check result] ************************************ -ok: [localhost] => { - "ansible_facts": { - "lock_check_result": { - "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "existing_deployment_hash": "none", - "existing_user_secrets_hash": "none", - "force_rerun": false, - "role_should_run": true, - "user_secrets_changed": true - } - }, - "changed": false -} - -TASK [lock_manager : Debug final lock check result] **************************** -ok: [localhost] => {} - -MSG: - -Final lock check result: - - role_should_run: True - - deployment_hash_changed: True - - user_secrets_changed: True - - force_rerun: False - - -TASK [lock_manager : Set current role (only if not already set)] *************** -ok: [localhost] => { - "ansible_facts": { - "current_role": "lock_manager" - }, - "changed": false -} - -TASK [lock_manager : Debug lock manager role execution] ************************ -ok: [localhost] => {} - -MSG: - -Lock manager role started - operation: check for role: lock_manager - -TASK [lock_manager : Ensure lock directory exists] ***************************** -changed: [localhost] => { - "changed": true, - "gid": 0, - "group": "root", - "mode": "0755", - "owner": "root", - "path": "/opt/vdi-setup", - "size": 6, - "state": "directory", - "uid": 0 -} - -TASK [lock_manager : Debug lock directory status] ****************************** -ok: [localhost] => {} - -MSG: - -Lock directory created: True - -TASK [lock_manager : Include check lock tasks] ********************************* -included: /tmp/vdi/roles/lock_manager/tasks/check_lock.yaml for localhost - -TASK [lock_manager : Check if lock file exists] ******************************** -ok: [localhost] => { - "changed": false, - "stat": { - "exists": false - } -} - -TASK [lock_manager : Debug lock file existence] ******************************** -ok: [localhost] => {} - -MSG: - -Lock file exists: False - -TASK [lock_manager : Load existing lock file] ********************************** -skipping: [localhost] => { - "changed": false, - "false_condition": "lock_file_stat.stat.exists", - "skip_reason": "Conditional result was False" -} - -TASK [lock_manager : Debug lock file load result] ****************************** -skipping: [localhost] => { - "false_condition": "debug | default(false) and lock_file_stat.stat.exists" -} - -TASK [lock_manager : Calculate current deployment hash] ************************ -ok: [localhost] => { - "ansible_facts": { - "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab" - }, - "changed": false -} - -TASK [lock_manager : Debug deployment hash calculation] ************************ -ok: [localhost] => {} - -MSG: - -Current deployment hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab -Deployment components: - - deployment_name: vdi-test-scott - - project_id: hpc-discovery-external - - vdi_tool: guacamole - - vnc_flavor: tigervnc - - user_provision: local_users - - vdi_webapp_port: 8081 - - vdi_resolution: 1920x1080 - - vdi_users count: 2 - - -TASK [lock_manager : Calculate current user secrets hash] ********************** -ok: [localhost] => { - "ansible_facts": { - "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" - }, - "changed": false -} - -TASK [lock_manager : Debug user secrets hash calculation] ********************** -ok: [localhost] => {} - -MSG: - -Current user secrets hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 -User configuration components: - - vdi_users (blueprint): [{"username": "alice", "port": 5901}, {"username": "bob", "port": 5902, "secret_name": "a-password-for-bob"}] - - -TASK [lock_manager : Set default values for missing lock file] ***************** -ok: [localhost] => { - "ansible_facts": { - "vdi_setup_status": { - "completed_roles": { - "base_os": { - "completed": false - }, - "lock_manager": { - "completed": false - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "none", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:27:41Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [], - "previous_users": [], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:27:41Z", - "user_secrets_hash": "none", - "users_updated": [] - } - } - }, - "changed": false -} - -TASK [lock_manager : Debug lock file status] *********************************** -ok: [localhost] => {} - -MSG: - -Lock file status: - - exists: False - - deployment_hash: none - - user_secrets_hash: none - - force_rerun: False - - -TASK [lock_manager : Check if lock file exists] ******************************** -ok: [localhost] => { - "ansible_facts": { - "lock_file_exists": false - }, - "changed": false -} - -TASK [lock_manager : Check if current role is completed] *********************** -ok: [localhost] => { - "ansible_facts": { - "current_role_completed": false - }, - "changed": false -} - -TASK [lock_manager : Check if deployment hash matches] ************************* -ok: [localhost] => { - "ansible_facts": { - "deployment_hash_matches": false - }, - "changed": false -} - -TASK [lock_manager : Check if force rerun is enabled] ************************** -ok: [localhost] => { - "ansible_facts": { - "force_rerun_enabled": false - }, - "changed": false -} - -TASK [lock_manager : Determine if role should run] ***************************** -ok: [localhost] => { - "ansible_facts": { - "role_should_run": true - }, - "changed": false -} - -TASK [lock_manager : Debug role should run calculation] ************************ -ok: [localhost] => {} - -MSG: - -Role should run calculation for lock_manager: - - not lock_file_exists: True - - not current_role_completed: True - - not deployment_hash_matches: True - - force_rerun_enabled: False - - role_should_run: True - - -TASK [lock_manager : Debug role execution decision] **************************** -ok: [localhost] => {} - -MSG: - -Role execution decision for lock_manager: - - current_role: lock_manager - - ansible_role_name: lock_manager - - role_should_run: True - - lock_file_exists: False - - current_role_completed: False - - deployment_hash_matches: False - - force_rerun_enabled: False - - -TASK [lock_manager : Check if user configuration has changed] ****************** -ok: [localhost] => { - "ansible_facts": { - "user_config_changed": true - }, - "changed": false -} - -TASK [lock_manager : Set user secrets changed flag (will be updated by secret_manager role)] *** -ok: [localhost] => { - "ansible_facts": { - "user_secrets_changed": true - }, - "changed": false -} - -TASK [lock_manager : Debug user secrets change detection] ********************** -ok: [localhost] => {} - -MSG: - -User configuration change detection: - - user_config_changed: True - - user_secrets_changed: True - - stored_hash: none - - current_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 - - -TASK [lock_manager : Set lock check result] ************************************ -ok: [localhost] => { - "ansible_facts": { - "lock_check_result": { - "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "existing_deployment_hash": "none", - "existing_user_secrets_hash": "none", - "force_rerun": false, - "role_should_run": true, - "user_secrets_changed": true - } - }, - "changed": false -} - -TASK [lock_manager : Debug final lock check result] **************************** -ok: [localhost] => {} - -MSG: - -Final lock check result: - - role_should_run: True - - deployment_hash_changed: True - - user_secrets_changed: True - - force_rerun: False - - -TASK [lock_manager : Include create lock tasks] ******************************** -skipping: [localhost] => { - "changed": false, - "false_condition": "lock_operation | default('check') == 'create'", - "skip_reason": "Conditional result was False" -} - -TASK [lock_manager : Include cleanup lock tasks] ******************************* -skipping: [localhost] => { - "changed": false, - "false_condition": "lock_operation | default('check') == 'cleanup'", - "skip_reason": "Conditional result was False" -} - -TASK [lock_manager : Debug lock manager role completion] *********************** -ok: [localhost] => {} - -MSG: - -Lock manager role completed - operation: check - -TASK [lock_manager : Mark lock_manager role as completed] ********************** -included: /tmp/vdi/roles/lock_manager/tasks/create_lock.yaml for localhost - -TASK [lock_manager : Debug create lock operation] ****************************** -ok: [localhost] => {} - -MSG: - -Creating/updating lock file for role: lock_manager - -TASK [lock_manager : Check if lock file exists] ******************************** -ok: [localhost] => { - "changed": false, - "stat": { - "exists": false - } -} - -TASK [lock_manager : Debug lock file existence] ******************************** -ok: [localhost] => {} - -MSG: - -Lock file exists: False - -TASK [lock_manager : Create initial lock file structure] *********************** -changed: [localhost] => { - "changed": true, - "checksum": "8479c862b92b39583c151fca822f92954c92eee0", - "dest": "/opt/vdi-setup/.vdi-lock.yaml", - "gid": 0, - "group": "root", - "md5sum": "e3c95caac8aea2fba82a6d39dec216d8", - "mode": "0644", - "owner": "root", - "size": 909, - "src": "/root/.ansible/tmp/ansible-tmp-1753370864.1744716-6171-78949705975838/source", - "state": "file", - "uid": 0 -} - -TASK [lock_manager : Debug lock file creation] ********************************* -ok: [localhost] => {} - -MSG: - -Lock file created: True - -TASK [lock_manager : Load existing lock file data] ***************************** -ok: [localhost] => { - "ansible_facts": { - "vdi_setup_status": { - "completed_roles": { - "base_os": { - "completed": false - }, - "lock_manager": { - "completed": false - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:27:41Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [], - "previous_users": [], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:27:41Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [] - } - } - }, - "ansible_included_var_files": [ - "/opt/vdi-setup/.vdi-lock.yaml" - ], - "changed": false -} - -TASK [lock_manager : Get current lock data] ************************************ -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": false - }, - "lock_manager": { - "completed": false - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:27:41Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [], - "previous_users": [], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:27:41Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [] - } - } - }, - "changed": false -} - -TASK [lock_manager : Update deployment hash] *********************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": false - }, - "lock_manager": { - "completed": false - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:27:41Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [], - "previous_users": [], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:27:41Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [] - } - } - }, - "changed": false -} - -TASK [lock_manager : Get current timestamp] ************************************ -changed: [localhost] => { - "changed": true, - "cmd": [ - "date", - "-u", - "+%Y-%m-%dT%H:%M:%SZ" - ], - "delta": "0:00:00.002395", - "end": "2025-07-24 15:27:44.996404", - "rc": 0, - "start": "2025-07-24 15:27:44.994009" -} - -STDOUT: - -2025-07-24T15:27:44Z - -TASK [lock_manager : Update last updated timestamp] **************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": false - }, - "lock_manager": { - "completed": false - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:27:44Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [], - "previous_users": [], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:27:41Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [] - } - } - }, - "changed": false -} - -TASK [lock_manager : Get current user secrets status] ************************** -ok: [localhost] => { - "ansible_facts": { - "current_user_secrets": { - "last_secret_check": "2025-07-24T15:27:41Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [] - } - }, - "changed": false -} - -TASK [lock_manager : Create user secrets update] ******************************* -ok: [localhost] => { - "ansible_facts": { - "user_secrets_update": { - "last_secret_check": "2025-07-24T15:27:44Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" - } - }, - "changed": false -} - -TASK [lock_manager : Update user secrets hash] ********************************* -ok: [localhost] => { - "ansible_facts": { - "updated_user_secrets": { - "last_secret_check": "2025-07-24T15:27:44Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [] - } - }, - "changed": false -} - -TASK [lock_manager : Apply user secrets update] ******************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": false - }, - "lock_manager": { - "completed": false - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:27:44Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [], - "previous_users": [], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:27:44Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [] - } - } - }, - "changed": false -} - -TASK [lock_manager : Create role status entry] ********************************* -ok: [localhost] => { - "ansible_facts": { - "role_status_entry": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - } - }, - "changed": false -} - -TASK [lock_manager : Debug role status entry] ********************************** -ok: [localhost] => {} - -MSG: - -Role status entry for lock_manager: - - completed: True - - completed_at: 2025-07-24T15:27:44Z - - -TASK [lock_manager : Debug current lock data before role update] *************** -ok: [localhost] => {} - -MSG: - -Current lock data before updating lock_manager: - - current_lock_data: completed_roles: - base_os: - completed: false - lock_manager: - completed: false - secret_manager: - completed: false - user_provision: - completed: false - vdi_tool: - completed: false - vnc: - completed: false -created_at: '2025-07-24T15:27:41Z' -deployment_hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab -deployment_name: vdi-test-scott -force_rerun: false -last_updated: '2025-07-24T15:27:44Z' -lock_version: '1.0' -setup_status: configuring -user_management: - current_users: [] - previous_users: [] - removed_users: [] -user_secrets_status: - last_secret_check: '2025-07-24T15:27:44Z' - user_secrets_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 - users_updated: [] - - - current_completed_roles: base_os: - completed: false -lock_manager: - completed: false -secret_manager: - completed: false -user_provision: - completed: false -vdi_tool: - completed: false -vnc: - completed: false - - - role_completed: True - - current_role: lock_manager - - -TASK [lock_manager : Get current completed roles] ****************************** -ok: [localhost] => { - "ansible_facts": { - "current_completed_roles": { - "base_os": { - "completed": false - }, - "lock_manager": { - "completed": false - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - } - }, - "changed": false -} - -TASK [lock_manager : Update completed roles] *********************************** -ok: [localhost] => { - "ansible_facts": { - "updated_completed_roles": { - "base_os": { - "completed": false - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - } - }, - "changed": false -} - -TASK [lock_manager : Update lock data with completed roles] ******************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": false - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:27:44Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [], - "previous_users": [], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:27:44Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [] - } - } - }, - "changed": false -} - -TASK [lock_manager : Debug updated lock data after role update] **************** -ok: [localhost] => {} - -MSG: - -Updated lock data after updating lock_manager: - - current_role: lock_manager - - role_completed: True - - completed_roles: {'base_os': {'completed': False}, 'lock_manager': {'completed': True, 'completed_at': '2025-07-24T15:27:44Z'}, 'secret_manager': {'completed': False}, 'user_provision': {'completed': False}, 'vnc': {'completed': False}, 'vdi_tool': {'completed': False}} - - -TASK [lock_manager : Get current users updated list] *************************** -ok: [localhost] => { - "ansible_facts": { - "current_users_updated": [] - }, - "changed": false -} - -TASK [lock_manager : Get new usernames list] *********************************** -ok: [localhost] => { - "ansible_facts": { - "new_usernames": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Combine users updated lists] ****************************** -ok: [localhost] => { - "ansible_facts": { - "combined_users_updated": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Debug user deduplication] ********************************* -ok: [localhost] => {} - -MSG: - -User deduplication for lock_manager: - - current_users_updated: [] - - new_usernames: ['alice', 'bob'] - - combined_users_updated: ['alice', 'bob'] - - -TASK [lock_manager : Get current user secrets for list update] ***************** -ok: [localhost] => { - "ansible_facts": { - "current_user_secrets_for_list": { - "last_secret_check": "2025-07-24T15:27:44Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [] - } - }, - "changed": false -} - -TASK [lock_manager : Update users updated list] ******************************** -ok: [localhost] => { - "ansible_facts": { - "updated_user_secrets_with_list": { - "last_secret_check": "2025-07-24T15:27:44Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Apply users updated list] ********************************* -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": false - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:27:44Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [], - "previous_users": [], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:27:44Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Get current user management data] ************************* -ok: [localhost] => { - "ansible_facts": { - "current_user_management": { - "current_users": [], - "previous_users": [], - "removed_users": [] - } - }, - "changed": false -} - -TASK [lock_manager : Get current usernames list] ******************************* -ok: [localhost] => { - "ansible_facts": { - "current_usernames": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Update user management data] ****************************** -ok: [localhost] => { - "ansible_facts": { - "updated_user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [], - "removed_users": [] - } - }, - "changed": false -} - -TASK [lock_manager : Apply user management update] ***************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": false - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:27:44Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:27:44Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Debug user management] ************************************ -ok: [localhost] => {} - -MSG: - -User management for lock_manager: - - current_users: ['alice', 'bob'] - - previous_users: [] - - removed_users: [] - - -TASK [lock_manager : Check if all roles are completed] ************************* -ok: [localhost] => { - "ansible_facts": { - "all_roles_completed": false - }, - "changed": false -} - -TASK [lock_manager : Update setup status based on completion] ****************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": false - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:27:44Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:27:44Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Debug setup status update] ******************************** -ok: [localhost] => {} - -MSG: - -Setup status update: - - all_roles_completed: False - - setup_status: configuring - - completed_roles_count: 1 - - total_roles_count: 6 - - -TASK [lock_manager : Write updated lock file] ********************************** -changed: [localhost] => { - "changed": true, - "checksum": "358a7f4095fbb49a0eedafd77e97040d1b9b5a26", - "dest": "/opt/vdi-setup/.vdi-lock.yaml", - "gid": 0, - "group": "root", - "md5sum": "2cf874286e9bf3b17fc771283955c695", - "mode": "0644", - "owner": "root", - "size": 879, - "src": "/root/.ansible/tmp/ansible-tmp-1753370865.96913-6272-170962703062563/source", - "state": "file", - "uid": 0 -} - -TASK [lock_manager : Debug VM metadata update] ********************************* -ok: [localhost] => {} - -MSG: - -VM metadata update info: - - hostname: vdi-test-scott-rocky-0 - - zone: us-central1-a - - lock_file_changed: True - - -TASK [lock_manager : Get lock file content as base64] ************************** -changed: [localhost] => { - "changed": true, - "cmd": "cat /opt/vdi-setup/.vdi-lock.yaml | base64 -w 0", - "delta": "0:00:00.005528", - "end": "2025-07-24 15:27:46.510774", - "rc": 0, - "start": "2025-07-24 15:27:46.505246" -} - -STDOUT: - -dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IGZhbHNlCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IGZhbHNlCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiBmYWxzZQogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogZmFsc2UKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiBmYWxzZQogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNToyNzo0NFonCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogY29uZmlndXJpbmcKICB1c2VyX21hbmFnZW1lbnQ6CiAgICBjdXJyZW50X3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHByZXZpb3VzX3VzZXJzOiBbXQogICAgcmVtb3ZlZF91c2VyczogW10KICB1c2VyX3NlY3JldHNfc3RhdHVzOgogICAgbGFzdF9zZWNyZXRfY2hlY2s6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHVzZXJfc2VjcmV0c19oYXNoOiA2YTZhMDJkM2QwOTkzM2NhMjgyYWUyYjAyMzM2MzQzYzlkNzM3MDFlNDViZDgxNDc1ZmJkOTJkY2Y5ZDJmM2I2CiAgICB1c2Vyc191cGRhdGVkOgogICAgLSBhbGljZQogICAgLSBib2IK - -TASK [lock_manager : Add VDI lock file content to VM metadata] ***************** -changed: [localhost] => { - "changed": true, - "cmd": [ - "gcloud", - "compute", - "instances", - "add-metadata", - "vdi-test-scott-rocky-0", - "--metadata", - "vdi-lock-content=dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IGZhbHNlCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IGZhbHNlCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiBmYWxzZQogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogZmFsc2UKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiBmYWxzZQogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNToyNzo0NFonCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogY29uZmlndXJpbmcKICB1c2VyX21hbmFnZW1lbnQ6CiAgICBjdXJyZW50X3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHByZXZpb3VzX3VzZXJzOiBbXQogICAgcmVtb3ZlZF91c2VyczogW10KICB1c2VyX3NlY3JldHNfc3RhdHVzOgogICAgbGFzdF9zZWNyZXRfY2hlY2s6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHVzZXJfc2VjcmV0c19oYXNoOiA2YTZhMDJkM2QwOTkzM2NhMjgyYWUyYjAyMzM2MzQzYzlkNzM3MDFlNDViZDgxNDc1ZmJkOTJkY2Y5ZDJmM2I2CiAgICB1c2Vyc191cGRhdGVkOgogICAgLSBhbGljZQogICAgLSBib2IK", - "--zone=us-central1-a" - ], - "delta": "0:00:04.460963", - "end": "2025-07-24 15:27:51.159269", - "failed_when_result": false, - "rc": 0, - "start": "2025-07-24 15:27:46.698306" -} - -STDERR: - -Updated [https://www.googleapis.com/compute/v1/projects/hpc-discovery-external/zones/us-central1-a/instances/vdi-test-scott-rocky-0]. - -TASK [lock_manager : Debug lock file write] ************************************ -ok: [localhost] => {} - -MSG: - -Lock file written: True - -TASK [lock_manager : Verify lock file integrity] ******************************* -ok: [localhost] => { - "changed": false, - "cmd": [ - "python3", - "-c", - "import yaml; yaml.safe_load(open('/opt/vdi-setup/.vdi-lock.yaml'))" - ], - "delta": "0:00:00.050900", - "end": "2025-07-24 15:27:51.433067", - "rc": 0, - "start": "2025-07-24 15:27:51.382167" -} - -TASK [lock_manager : Debug lock file validation] ******************************* -ok: [localhost] => {} - -MSG: - -Lock file validation: True - -TASK [lock_manager : Check if VDI monitor service exists] ********************** -ok: [localhost] => { - "changed": false, - "stat": { - "exists": false - } -} - -TASK [lock_manager : Debug VDI monitor setup conditions] *********************** -ok: [localhost] => {} - -MSG: - -VDI monitor setup conditions: - - current_role: lock_manager - - vdi_monitor_service_exists: False - - lock_file_creation.changed: True - - will_setup_monitor: True - - -TASK [lock_manager : Setup VDI monitoring service (first time only)] *********** -included: /tmp/vdi/roles/lock_manager/tasks/monitor_setup.yaml for localhost - -TASK [lock_manager : Create VDI monitor script directory] ********************** -ok: [localhost] => { - "changed": false, - "gid": 0, - "group": "root", - "mode": "0755", - "owner": "root", - "path": "/usr/local/bin", - "size": 6, - "state": "directory", - "uid": 0 -} - -TASK [lock_manager : Copy VDI monitor script] ********************************** -changed: [localhost] => { - "changed": true, - "checksum": "49f5302fb57094d20c194a1f7379da9d16209de7", - "dest": "/usr/local/bin/vdi-monitor.sh", - "gid": 0, - "group": "root", - "md5sum": "62d4f0ae8c6876764d097330ba1393cb", - "mode": "0755", - "owner": "root", - "size": 5348, - "src": "/root/.ansible/tmp/ansible-tmp-1753370871.9519365-6406-123074483660618/source", - "state": "file", - "uid": 0 -} - -TASK [lock_manager : Copy VDI monitor systemd service] ************************* -changed: [localhost] => { - "changed": true, - "checksum": "f6de22707f8d9248014f679a2f1930451e2bf635", - "dest": "/etc/systemd/system/vdi-monitor.service", - "gid": 0, - "group": "root", - "md5sum": "03b2639860cf4c6a87c36ae7f885909d", - "mode": "0644", - "owner": "root", - "size": 283, - "src": "/root/.ansible/tmp/ansible-tmp-1753370872.3131151-6428-31276884095225/source", - "state": "file", - "uid": 0 -} - -TASK [lock_manager : Create VDI monitor log directory] ************************* -ok: [localhost] => { - "changed": false, - "gid": 0, - "group": "root", - "mode": "0755", - "owner": "root", - "path": "/var/log", - "size": 4096, - "state": "directory", - "uid": 0 -} - -TASK [lock_manager : Create VDI monitor log files] ***************************** -changed: [localhost] => (item=/var/log/vdi-monitor.log) => { - "ansible_loop_var": "item", - "changed": true, - "dest": "/var/log/vdi-monitor.log", - "gid": 0, - "group": "root", - "item": "/var/log/vdi-monitor.log", - "mode": "0644", - "owner": "root", - "size": 0, - "state": "file", - "uid": 0 -} -changed: [localhost] => (item=/var/log/ansible-vdi-reconfig.log) => { - "ansible_loop_var": "item", - "changed": true, - "dest": "/var/log/ansible-vdi-reconfig.log", - "gid": 0, - "group": "root", - "item": "/var/log/ansible-vdi-reconfig.log", - "mode": "0644", - "owner": "root", - "size": 0, - "state": "file", - "uid": 0 -} - -TASK [lock_manager : Reload systemd daemon] ************************************ -ok: [localhost] => { - "changed": false, - "name": null, - "status": {} -} - -TASK [lock_manager : Enable VDI monitor service] ******************************* -changed: [localhost] => { - "changed": true, - "enabled": true, - "name": "vdi-monitor", - "state": "started", - "status": { - "ActiveEnterTimestampMonotonic": "0", - "ActiveExitTimestampMonotonic": "0", - "ActiveState": "inactive", - "After": "systemd-journald.socket network.target system.slice sysinit.target basic.target", - "AllowIsolate": "no", - "AllowedCPUs": "", - "AllowedMemoryNodes": "", - "AmbientCapabilities": "", - "AssertResult": "no", - "AssertTimestampMonotonic": "0", - "Before": "shutdown.target", - "BlockIOAccounting": "no", - "BlockIOWeight": "[not set]", - "CPUAccounting": "no", - "CPUAffinity": "", - "CPUAffinityFromNUMA": "no", - "CPUQuotaPerSecUSec": "infinity", - "CPUQuotaPeriodUSec": "infinity", - "CPUSchedulingPolicy": "0", - "CPUSchedulingPriority": "0", - "CPUSchedulingResetOnFork": "no", - "CPUShares": "[not set]", - "CPUUsageNSec": "[not set]", - "CPUWeight": "[not set]", - "CacheDirectoryMode": "0755", - "CanFreeze": "yes", - "CanIsolate": "no", - "CanReload": "no", - "CanStart": "yes", - "CanStop": "yes", - "CapabilityBoundingSet": "cap_chown cap_dac_override cap_dac_read_search cap_fowner cap_fsetid cap_kill cap_setgid cap_setuid cap_setpcap cap_linux_immutable cap_net_bind_service cap_net_broadcast cap_net_admin cap_net_raw cap_ipc_lock cap_ipc_owner cap_sys_module cap_sys_rawio cap_sys_chroot cap_sys_ptrace cap_sys_pacct cap_sys_admin cap_sys_boot cap_sys_nice cap_sys_resource cap_sys_time cap_sys_tty_config cap_mknod cap_lease cap_audit_write cap_audit_control cap_setfcap cap_mac_override cap_mac_admin cap_syslog cap_wake_alarm cap_block_suspend cap_audit_read cap_perfmon cap_bpf", - "CollectMode": "inactive", - "ConditionResult": "no", - "ConditionTimestampMonotonic": "0", - "ConfigurationDirectoryMode": "0755", - "Conflicts": "shutdown.target", - "ControlPID": "0", - "DefaultDependencies": "yes", - "DefaultMemoryLow": "0", - "DefaultMemoryMin": "0", - "Delegate": "no", - "Description": "VDI Configuration Monitor", - "DevicePolicy": "auto", - "DynamicUser": "no", - "EffectiveCPUs": "", - "EffectiveMemoryNodes": "", - "ExecMainCode": "0", - "ExecMainExitTimestampMonotonic": "0", - "ExecMainPID": "0", - "ExecMainStartTimestampMonotonic": "0", - "ExecMainStatus": "0", - "ExecStart": "{ path=/usr/local/bin/vdi-monitor.sh ; argv[]=/usr/local/bin/vdi-monitor.sh ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", - "FailureAction": "none", - "FileDescriptorStoreMax": "0", - "FragmentPath": "/etc/systemd/system/vdi-monitor.service", - "FreezerState": "running", - "GID": "[not set]", - "Group": "root", - "GuessMainPID": "yes", - "IOAccounting": "no", - "IOSchedulingClass": "0", - "IOSchedulingPriority": "0", - "IOWeight": "[not set]", - "IPAccounting": "no", - "IPEgressBytes": "18446744073709551615", - "IPEgressPackets": "18446744073709551615", - "IPIngressBytes": "18446744073709551615", - "IPIngressPackets": "18446744073709551615", - "Id": "vdi-monitor.service", - "IgnoreOnIsolate": "no", - "IgnoreSIGPIPE": "yes", - "InactiveEnterTimestampMonotonic": "0", - "InactiveExitTimestampMonotonic": "0", - "JobRunningTimeoutUSec": "infinity", - "JobTimeoutAction": "none", - "JobTimeoutUSec": "infinity", - "KeyringMode": "private", - "KillMode": "control-group", - "KillSignal": "15", - "LimitAS": "infinity", - "LimitASSoft": "infinity", - "LimitCORE": "infinity", - "LimitCORESoft": "0", - "LimitCPU": "infinity", - "LimitCPUSoft": "infinity", - "LimitDATA": "infinity", - "LimitDATASoft": "infinity", - "LimitFSIZE": "infinity", - "LimitFSIZESoft": "infinity", - "LimitLOCKS": "infinity", - "LimitLOCKSSoft": "infinity", - "LimitMEMLOCK": "65536", - "LimitMEMLOCKSoft": "65536", - "LimitMSGQUEUE": "819200", - "LimitMSGQUEUESoft": "819200", - "LimitNICE": "0", - "LimitNICESoft": "0", - "LimitNOFILE": "262144", - "LimitNOFILESoft": "1024", - "LimitNPROC": "127383", - "LimitNPROCSoft": "127383", - "LimitRSS": "infinity", - "LimitRSSSoft": "infinity", - "LimitRTPRIO": "0", - "LimitRTPRIOSoft": "0", - "LimitRTTIME": "infinity", - "LimitRTTIMESoft": "infinity", - "LimitSIGPENDING": "127383", - "LimitSIGPENDINGSoft": "127383", - "LimitSTACK": "infinity", - "LimitSTACKSoft": "8388608", - "LoadState": "loaded", - "LockPersonality": "no", - "LogLevelMax": "-1", - "LogRateLimitBurst": "0", - "LogRateLimitIntervalUSec": "0", - "LogsDirectoryMode": "0755", - "MainPID": "0", - "MemoryAccounting": "yes", - "MemoryCurrent": "[not set]", - "MemoryDenyWriteExecute": "no", - "MemoryHigh": "infinity", - "MemoryLimit": "infinity", - "MemoryLow": "0", - "MemoryMax": "infinity", - "MemoryMin": "0", - "MemorySwapMax": "infinity", - "MountAPIVFS": "no", - "MountFlags": "", - "NFileDescriptorStore": "0", - "NRestarts": "0", - "NUMAMask": "", - "NUMAPolicy": "n/a", - "Names": "vdi-monitor.service", - "NeedDaemonReload": "no", - "Nice": "0", - "NoNewPrivileges": "no", - "NonBlocking": "no", - "NotifyAccess": "none", - "OOMScoreAdjust": "0", - "OnFailureJobMode": "replace", - "PermissionsStartOnly": "no", - "Perpetual": "no", - "PrivateDevices": "no", - "PrivateMounts": "no", - "PrivateNetwork": "no", - "PrivateTmp": "no", - "PrivateUsers": "no", - "ProtectControlGroups": "no", - "ProtectHome": "no", - "ProtectKernelModules": "no", - "ProtectKernelTunables": "no", - "ProtectSystem": "no", - "RefuseManualStart": "no", - "RefuseManualStop": "no", - "RemainAfterExit": "no", - "RemoveIPC": "no", - "Requires": "sysinit.target system.slice", - "Restart": "always", - "RestartUSec": "30s", - "RestrictNamespaces": "no", - "RestrictRealtime": "no", - "RestrictSUIDSGID": "no", - "Result": "success", - "RootDirectoryStartOnly": "no", - "RuntimeDirectoryMode": "0755", - "RuntimeDirectoryPreserve": "no", - "RuntimeMaxUSec": "infinity", - "SameProcessGroup": "no", - "SecureBits": "0", - "SendSIGHUP": "no", - "SendSIGKILL": "yes", - "Slice": "system.slice", - "StandardError": "journal", - "StandardInput": "null", - "StandardInputData": "", - "StandardOutput": "journal", - "StartLimitAction": "none", - "StartLimitBurst": "5", - "StartLimitIntervalUSec": "10s", - "StartupBlockIOWeight": "[not set]", - "StartupCPUShares": "[not set]", - "StartupCPUWeight": "[not set]", - "StartupIOWeight": "[not set]", - "StateChangeTimestampMonotonic": "0", - "StateDirectoryMode": "0755", - "StatusErrno": "0", - "StopWhenUnneeded": "no", - "SubState": "dead", - "SuccessAction": "none", - "SyslogFacility": "3", - "SyslogLevel": "6", - "SyslogLevelPrefix": "yes", - "SyslogPriority": "30", - "SystemCallErrorNumber": "0", - "TTYReset": "no", - "TTYVHangup": "no", - "TTYVTDisallocate": "no", - "TasksAccounting": "yes", - "TasksCurrent": "[not set]", - "TasksMax": "203813", - "TimeoutStartUSec": "1min 30s", - "TimeoutStopUSec": "1min 30s", - "TimerSlackNSec": "50000", - "Transient": "no", - "Type": "simple", - "UID": "[not set]", - "UMask": "0022", - "UnitFilePreset": "disabled", - "UnitFileState": "disabled", - "User": "root", - "UtmpMode": "init", - "Wants": "network.target", - "WatchdogTimestampMonotonic": "0", - "WatchdogUSec": "0" - } -} - -TASK [lock_manager : Debug VDI monitor setup] ********************************** -ok: [localhost] => {} - -MSG: - -VDI monitor service setup complete: - - Service: vdi-monitor.service - - Script: /usr/local/bin/vdi-monitor.sh - - Logs: /var/log/vdi-monitor.log, /var/log/ansible-vdi-reconfig.log - - Service enabled and started - - -TASK [lock_manager : Fail if lock file is invalid] ***************************** -skipping: [localhost] => { - "changed": false, - "false_condition": "lock_file_validation.rc != 0", - "skip_reason": "Conditional result was False" -} - -TASK [base_os : DEBUG About to import lock_manager for base_os] **************** -ok: [localhost] => {} - -MSG: - -About to import lock_manager for base_os - -TASK [base_os : Set current role for lock manager check] *********************** -ok: [localhost] => { - "ansible_facts": { - "current_role": "base_os" - }, - "changed": false -} - -TASK [lock_manager : Check if lock file exists] ******************************** -ok: [localhost] => { - "changed": false, - "stat": { - "atime": 1753370866.5072315, - "attr_flags": "", - "attributes": [], - "block_size": 4096, - "blocks": 8, - "charset": "us-ascii", - "checksum": "358a7f4095fbb49a0eedafd77e97040d1b9b5a26", - "ctime": 1753370866.2832148, - "dev": 2050, - "device_type": 0, - "executable": false, - "exists": true, - "gid": 0, - "gr_name": "root", - "inode": 891734, - "isblk": false, - "ischr": false, - "isdir": false, - "isfifo": false, - "isgid": false, - "islnk": false, - "isreg": true, - "issock": false, - "isuid": false, - "mimetype": "text/plain", - "mode": "0644", - "mtime": 1753370866.137204, - "nlink": 1, - "path": "/opt/vdi-setup/.vdi-lock.yaml", - "pw_name": "root", - "readable": true, - "rgrp": true, - "roth": true, - "rusr": true, - "size": 879, - "uid": 0, - "version": "3120075969", - "wgrp": false, - "woth": false, - "writeable": true, - "wusr": true, - "xgrp": false, - "xoth": false, - "xusr": false - } -} - -TASK [lock_manager : Debug lock file existence] ******************************** -ok: [localhost] => {} - -MSG: - -Lock file exists: True - -TASK [lock_manager : Load existing lock file] ********************************** -ok: [localhost] => { - "ansible_facts": { - "vdi_setup_status": { - "completed_roles": { - "base_os": { - "completed": false - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:27:44Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:27:44Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "ansible_included_var_files": [ - "/opt/vdi-setup/.vdi-lock.yaml" - ], - "changed": false -} - -TASK [lock_manager : Debug lock file load result] ****************************** -ok: [localhost] => {} - -MSG: - -Lock file loaded successfully: True - -TASK [lock_manager : Calculate current deployment hash] ************************ -ok: [localhost] => { - "ansible_facts": { - "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab" - }, - "changed": false -} - -TASK [lock_manager : Debug deployment hash calculation] ************************ -ok: [localhost] => {} - -MSG: - -Current deployment hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab -Deployment components: - - deployment_name: vdi-test-scott - - project_id: hpc-discovery-external - - vdi_tool: guacamole - - vnc_flavor: tigervnc - - user_provision: local_users - - vdi_webapp_port: 8081 - - vdi_resolution: 1920x1080 - - vdi_users count: 2 - - -TASK [lock_manager : Calculate current user secrets hash] ********************** -ok: [localhost] => { - "ansible_facts": { - "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" - }, - "changed": false -} - -TASK [lock_manager : Debug user secrets hash calculation] ********************** -ok: [localhost] => {} - -MSG: - -Current user secrets hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 -User configuration components: - - vdi_users (blueprint): [{"username": "alice", "port": 5901}, {"username": "bob", "port": 5902, "secret_name": "a-password-for-bob"}] - - -TASK [lock_manager : Set default values for missing lock file] ***************** -skipping: [localhost] => { - "changed": false, - "false_condition": "not lock_file_stat.stat.exists", - "skip_reason": "Conditional result was False" -} - -TASK [lock_manager : Debug lock file status] *********************************** -ok: [localhost] => {} - -MSG: - -Lock file status: - - exists: True - - deployment_hash: none - - user_secrets_hash: none - - force_rerun: False - - -TASK [lock_manager : Check if lock file exists] ******************************** -ok: [localhost] => { - "ansible_facts": { - "lock_file_exists": true - }, - "changed": false -} - -TASK [lock_manager : Check if current role is completed] *********************** -ok: [localhost] => { - "ansible_facts": { - "current_role_completed": false - }, - "changed": false -} - -TASK [lock_manager : Check if deployment hash matches] ************************* -ok: [localhost] => { - "ansible_facts": { - "deployment_hash_matches": false - }, - "changed": false -} - -TASK [lock_manager : Check if force rerun is enabled] ************************** -ok: [localhost] => { - "ansible_facts": { - "force_rerun_enabled": false - }, - "changed": false -} - -TASK [lock_manager : Determine if role should run] ***************************** -ok: [localhost] => { - "ansible_facts": { - "role_should_run": true - }, - "changed": false -} - -TASK [lock_manager : Debug role should run calculation] ************************ -ok: [localhost] => {} - -MSG: - -Role should run calculation for base_os: - - not lock_file_exists: False - - not current_role_completed: True - - not deployment_hash_matches: True - - force_rerun_enabled: False - - role_should_run: True - - -TASK [lock_manager : Debug role execution decision] **************************** -ok: [localhost] => {} - -MSG: - -Role execution decision for base_os: - - current_role: base_os - - ansible_role_name: lock_manager - - role_should_run: True - - lock_file_exists: True - - current_role_completed: False - - deployment_hash_matches: False - - force_rerun_enabled: False - - -TASK [lock_manager : Check if user configuration has changed] ****************** -ok: [localhost] => { - "ansible_facts": { - "user_config_changed": true - }, - "changed": false -} - -TASK [lock_manager : Set user secrets changed flag (will be updated by secret_manager role)] *** -ok: [localhost] => { - "ansible_facts": { - "user_secrets_changed": true - }, - "changed": false -} - -TASK [lock_manager : Debug user secrets change detection] ********************** -ok: [localhost] => {} - -MSG: - -User configuration change detection: - - user_config_changed: True - - user_secrets_changed: True - - stored_hash: none - - current_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 - - -TASK [lock_manager : Set lock check result] ************************************ -ok: [localhost] => { - "ansible_facts": { - "lock_check_result": { - "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "existing_deployment_hash": "none", - "existing_user_secrets_hash": "none", - "force_rerun": false, - "role_should_run": true, - "user_secrets_changed": true - } - }, - "changed": false -} - -TASK [lock_manager : Debug final lock check result] **************************** -ok: [localhost] => {} - -MSG: - -Final lock check result: - - role_should_run: True - - deployment_hash_changed: True - - user_secrets_changed: True - - force_rerun: False - - -TASK [base_os : DEBUG Finished import_role for base_os] ************************ -ok: [localhost] => {} - -MSG: - -Finished import_role for base_os - -TASK [base_os : Set role_should_run from lock check result] ******************** -ok: [localhost] => { - "ansible_facts": { - "role_should_run": true - }, - "changed": false -} - -TASK [base_os : Skip base_os tasks if role should not run] ********************* -skipping: [localhost] => { - "false_condition": "not role_should_run" -} - -TASK [base_os : Gather distribution facts] ************************************* -ok: [localhost] - -TASK [base_os : Check if NVIDIA GPU is present] ******************************** -changed: [localhost] => { - "changed": true, - "cmd": "lspci | grep -i nvidia", - "delta": "0:00:00.013393", - "end": "2025-07-24 15:27:55.866012", - "failed_when_result": false, - "rc": 1, - "start": "2025-07-24 15:27:55.852619" -} - -MSG: - -non-zero return code - -TASK [base_os : Include Ubuntu-specific tasks] ********************************* -skipping: [localhost] => { - "changed": false, - "false_condition": "ansible_facts.distribution == 'Ubuntu'", - "skip_reason": "Conditional result was False" -} - -TASK [base_os : Include Rocky Linux specific tasks] **************************** -included: /tmp/vdi/roles/base_os/tasks/rocky.yaml for localhost - -TASK [base_os : Set Rocky Linux specific variables] **************************** -ok: [localhost] => { - "ansible_facts": { - "gui_packages": [ - "@Xfce", - "@base-x" - ], - "vnc_pkg": "tigervnc-server", - "xorg_dev_packages": [ - "pkgconfig", - "xorg-x11-server-devel", - "xorg-x11-util-macros", - "libglvnd-devel" - ] - }, - "changed": false -} - -TASK [base_os : Update repository cache & install EPEL (Rocky Linux only)] ***** -ok: [localhost] => { - "changed": false, - "rc": 0, - "results": [] -} - -MSG: - -Nothing to do - -TASK [base_os : Install desktop prerequisites & VNC package (Rocky Linux)] ***** -changed: [localhost] => { - "changed": true, - "rc": 0, - "results": [ - "Group xfce-desktop installed.", - "Group base-x installed.", - "Installed: garcon-4.16.1-1.el8.x86_64", - "Installed: desktop-file-utils-0.26-1.el8.x86_64", - "Installed: libfontenc-devel-1.1.3-8.el8.x86_64", - "Installed: accountsservice-0.6.55-4.el8.x86_64", - "Installed: vte-profile-0.52.4-2.el8.x86_64", - "Installed: accountsservice-libs-0.6.55-4.el8.x86_64", - "Installed: vte291-0.52.4-2.el8.x86_64", - "Installed: bluez-5.63-5.el8_10.x86_64", - "Installed: bluez-libs-5.63-5.el8_10.x86_64", - "Installed: adwaita-cursor-theme-3.28.0-3.el8.noarch", - "Installed: bluez-obexd-5.63-5.el8_10.x86_64", - "Installed: adwaita-gtk2-theme-3.22.3-4.el8.x86_64", - "Installed: adwaita-icon-theme-3.28.0-3.el8.noarch", - "Installed: ostree-libs-2022.2-8.el8.x86_64", - "Installed: bolt-0.9.1-1.el8.x86_64", - "Installed: libasyncns-0.8-14.el8.x86_64", - "Installed: alsa-lib-1.2.10-2.el8.x86_64", - "Installed: python3-pysocks-1.6.8-3.el8.noarch", - "Installed: libavc1394-0.5.4-7.el8.x86_64", - "Installed: wavpack-5.1.0-16.el8.x86_64", - "Installed: thunar-archive-plugin-0.4.0-26.el8.x86_64", - "Installed: thunar-volman-4.16.0-3.el8.x86_64", - "Installed: poppler-20.11.0-12.el8_10.x86_64", - "Installed: poppler-data-0.4.9-2.el8.noarch", - "Installed: python3-requests-2.20.0-5.el8_10.noarch", - "Installed: xfce-polkit-0.3-3.el8.x86_64", - "Installed: xfce4-appfinder-4.16.1-3.el8.x86_64", - "Installed: poppler-glib-20.11.0-12.el8_10.x86_64", - "Installed: libquvi-0.9.4-12.el8.x86_64", - "Installed: libquvi-scripts-0.9.20131130-9.el8.noarch", - "Installed: hunspell-1.6.2-1.el8.x86_64", - "Installed: libraw1394-2.1.2-5.el8.x86_64", - "Installed: pangomm-2.40.1-6.el8.x86_64", - "Installed: webkit2gtk3-2.48.3-1.el8_10.x86_64", - "Installed: llvm-compat-libs-17.0.6-3.module+el8.10.0+1875+4f0b06db.x86_64", - "Installed: libical-3.0.3-3.el8.x86_64", - "Installed: xfce4-panel-4.16.3-1.el8.x86_64", - "Installed: xfce4-power-manager-4.16.0-1.el8.x86_64", - "Installed: xfce4-pulseaudio-plugin-0.4.3-3.el8.x86_64", - "Installed: bzip2-devel-1.0.6-28.el8_10.x86_64", - "Installed: sbc-1.3-9.el8.x86_64", - "Installed: xfce4-screensaver-4.16.0-3.el8.x86_64", - "Installed: pavucontrol-3.0-11.el8.x86_64", - "Installed: xfce4-session-4.16.0-4.el8.x86_64", - "Installed: xfce4-settings-4.16.5-2.el8.x86_64", - "Installed: gnome-themes-standard-3.22.3-4.el8.x86_64", - "Installed: xfce4-terminal-1.0.4-1.el8.x86_64", - "Installed: hunspell-en-0.20140811.1-12.el8.noarch", - "Installed: hunspell-en-GB-0.20140811.1-12.el8.noarch", - "Installed: hunspell-en-US-0.20140811.1-12.el8.noarch", - "Installed: mobile-broadband-provider-info-20210805-1.el8.noarch", - "Installed: xfconf-4.16.0-1.el8.x86_64", - "Installed: xfdesktop-4.16.0-3.el8.x86_64", - "Installed: webkit2gtk3-jsc-2.48.3-1.el8_10.x86_64", - "Installed: xfwm4-4.16.1-1.el8.x86_64", - "Installed: libcanberra-0.30-18.el8.x86_64", - "Installed: webrtc-audio-processing-0.3-10.el8.x86_64", - "Installed: python3-urllib3-1.24.2-8.el8_10.noarch", - "Installed: libmousepad0-0.5.6-1.el8.x86_64", - "Installed: libcanberra-gtk3-0.30-18.el8.x86_64", - "Installed: crda-3.18_2020.04.29-1.el8.noarch", - "Installed: cairo-gobject-1.15.12-6.el8.x86_64", - "Installed: tumbler-0.2.7-1.el8.x86_64", - "Installed: woff2-1.0.2-5.el8.x86_64", - "Installed: cairomm-1.12.0-8.el8.x86_64", - "Installed: libmodman-2.0.1-17.el8.x86_64", - "Installed: libXvMC-1.0.12-1.el8.x86_64", - "Installed: aspell-12:0.60.6.1-22.el8.x86_64", - "Installed: libdbusmenu-16.04.0-12.el8.x86_64", - "Installed: libdbusmenu-gtk3-16.04.0-12.el8.x86_64", - "Installed: xcb-util-0.4.0-10.el8.x86_64", - "Installed: python3-chardet-3.0.4-7.el8.noarch", - "Installed: lua-expat-1.3.0-12.el8.1.x86_64", - "Installed: gsettings-desktop-schemas-3.32.0-6.el8.x86_64", - "Installed: libdmx-1.1.4-3.el8.x86_64", - "Installed: rocky-logos-86.3-1.el8.x86_64", - "Installed: lua-socket-3.0-0.17.rc1.el8.x86_64", - "Installed: libdrm-2.4.115-2.el8.x86_64", - "Installed: xdg-desktop-portal-1.12.6-1.el8.x86_64", - "Installed: libdrm-devel-2.4.115-2.el8.x86_64", - "Installed: cheese-libs-2:3.28.0-4.el8_6.x86_64", - "Installed: libdv-1.0.0-27.el8.x86_64", - "Installed: libshout-2.2.2-19.el8.x86_64", - "Installed: libdvdnav-5.0.3-8.el8.x86_64", - "Installed: libsigc++20-2.10.0-6.el8.x86_64", - "Installed: libdvdread-5.0.3-9.el8.x86_64", - "Installed: xdg-desktop-portal-gtk-1.12.0-1.el8.x86_64", - "Installed: samba-common-4.19.4-9.el8_10.noarch", - "Installed: libepoxy-1.5.8-1.el8.x86_64", - "Installed: python3-pycurl-7.43.0.2-4.el8.x86_64", - "Installed: xorg-x11-drv-ati-19.1.0-1.el8.x86_64", - "Installed: dejavu-fonts-common-2.35-7.el8.noarch", - "Installed: libsndfile-1.0.28-16.el8_10.x86_64", - "Installed: xorg-x11-drv-evdev-2.10.6-2.el8.x86_64", - "Installed: dejavu-sans-mono-fonts-2.35-7.el8.noarch", - "Installed: xorg-x11-drv-fbdev-0.5.0-2.el8.x86_64", - "Installed: flac-libs-1.3.2-11.el8.x86_64", - "Installed: libevdev-1.10.0-1.el8.x86_64", - "Installed: nm-connection-editor-1.26.0-1.el8.x86_64", - "Installed: flatpak-1.12.9-3.el8_10.x86_64", - "Installed: libexif-0.6.22-5.el8_3.x86_64", - "Installed: xorg-x11-drv-intel-2.99.917-41.20210115.el8.x86_64", - "Installed: xorg-x11-drv-libinput-0.29.0-1.el8.x86_64", - "Installed: xorg-x11-drv-nouveau-1:1.0.15-4.el8.1.x86_64", - "Installed: xorg-x11-drv-qxl-0.1.5-11.el8.x86_64", - "Installed: xorg-x11-drv-vesa-2.4.0-3.el8.x86_64", - "Installed: xorg-x11-drv-vmware-13.2.1-8.el8.x86_64", - "Installed: xorg-x11-drv-wacom-0.38.0-1.el8.x86_64", - "Installed: xorg-x11-drv-wacom-serial-support-0.38.0-1.el8.x86_64", - "Installed: flatpak-selinux-1.12.9-3.el8_10.noarch", - "Installed: libsrtp-1.5.4-8.el8.x86_64", - "Installed: sound-theme-freedesktop-0.8-9.el8.noarch", - "Installed: flatpak-session-helper-1.12.9-3.el8_10.x86_64", - "Installed: soundtouch-2.0.0-3.el8.x86_64", - "Installed: iw-5.19-1.el8.1.x86_64", - "Installed: libpciaccess-0.14-1.el8.x86_64", - "Installed: libgdata-0.17.9-4.el8.x86_64", - "Installed: at-spi2-atk-2.26.2-1.el8.x86_64", - "Installed: libpng-devel-2:1.6.34-5.el8.x86_64", - "Installed: at-spi2-core-2.28.0-1.el8.x86_64", - "Installed: xorg-x11-proto-devel-2020.1-3.el8.noarch", - "Installed: libproxy-0.4.15-5.5.el8_10.x86_64", - "Installed: hyphen-2.8.8-9.el8.x86_64", - "Installed: speex-1.2.0-1.el8.x86_64", - "Installed: libglvnd-1:1.3.4-2.el8.x86_64", - "Installed: speexdsp-1.2-0.13.rc3.el8.x86_64", - "Installed: libglvnd-core-devel-1:1.3.4-2.el8.x86_64", - "Installed: samba-client-libs-4.19.4-9.el8_10.x86_64", - "Installed: atkmm-2.24.2-7.el8.x86_64", - "Installed: libglvnd-devel-1:1.3.4-2.el8.x86_64", - "Installed: libglvnd-egl-1:1.3.4-2.el8.x86_64", - "Installed: libglvnd-gles-1:1.3.4-2.el8.x86_64", - "Installed: clutter-1.26.2-8.el8.x86_64", - "Installed: libglvnd-glx-1:1.3.4-2.el8.x86_64", - "Installed: libtheora-1:1.1.1-21.el8.x86_64", - "Installed: libglvnd-opengl-1:1.3.4-2.el8.x86_64", - "Installed: clutter-gst3-3.0.26-1.el8.x86_64", - "Installed: libgnomekbd-3.26.0-4.el8.x86_64", - "Installed: clutter-gtk-1.8.4-3.el8.x86_64", - "Installed: tigervnc-license-1.15.0-7.el8_10.noarch", - "Installed: xorg-x11-server-Xorg-1.20.11-26.el8_10.x86_64", - "Installed: gcr-3.28.0-1.el8.x86_64", - "Installed: xorg-x11-server-Xwayland-21.1.3-18.el8_10.x86_64", - "Installed: tigervnc-selinux-1.15.0-7.el8_10.noarch", - "Installed: xorg-x11-server-common-1.20.11-26.el8_10.x86_64", - "Installed: p11-kit-server-0.23.22-2.el8.x86_64", - "Installed: xorg-x11-utils-7.5-28.el8.x86_64", - "Installed: xorg-x11-xauth-1:1.0.9-12.el8.x86_64", - "Installed: samba-common-libs-4.19.4-9.el8_10.x86_64", - "Installed: xorg-x11-xinit-1.3.4-18.el8.x86_64", - "Installed: xorg-x11-xinit-session-1.3.4-18.el8.x86_64", - "Installed: xorg-x11-xkb-utils-7.7-28.el8.x86_64", - "Installed: tigervnc-server-1.15.0-7.el8_10.x86_64", - "Installed: libgtop2-2.38.0-3.el8.x86_64", - "Installed: gdm-1:40.0-27.el8.x86_64", - "Installed: tigervnc-server-minimal-1.15.0-7.el8_10.x86_64", - "Installed: startup-notification-0.12-15.el8.x86_64", - "Installed: cogl-1.22.2-10.el8.x86_64", - "Installed: color-filesystem-1-20.el8.noarch", - "Installed: colord-1.4.2-1.el8.x86_64", - "Installed: libusbmuxd-1.0.10-9.el8.x86_64", - "Installed: colord-gtk-0.1.26-8.el8.x86_64", - "Installed: colord-libs-1.4.2-1.el8.x86_64", - "Installed: libv4l-1.14.2-3.el8.x86_64", - "Installed: keybinder3-0.3.2-4.el8.x86_64", - "Installed: zenity-3.28.1-2.el8.x86_64", - "Installed: ibus-1.5.19-15.el8_10.x86_64", - "Installed: geoclue2-2.5.5-2.el8.x86_64", - "Installed: ibus-gtk2-1.5.19-15.el8_10.x86_64", - "Installed: geoclue2-libs-2.5.5-2.el8.x86_64", - "Installed: ibus-gtk3-1.5.19-15.el8_10.x86_64", - "Installed: geocode-glib-3.26.0-4.el8_10.x86_64", - "Installed: ibus-libs-1.5.19-15.el8_10.x86_64", - "Installed: lame-libs-3.100-6.el8.x86_64", - "Installed: libgweather-3.28.2-4.el8.x86_64", - "Installed: ibus-setup-1.5.19-15.el8_10.noarch", - "Installed: libsmbclient-4.19.4-9.el8_10.x86_64", - "Installed: mousepad-0.5.6-1.el8.x86_64", - "Installed: switcheroo-control-1.1-5.el8.x86_64", - "Installed: mesa-dri-drivers-23.1.4-4.el8_10.x86_64", - "Installed: libiec61883-1.2.0-18.el8.x86_64", - "Installed: mesa-filesystem-23.1.4-4.el8_10.x86_64", - "Installed: libsoup-2.62.3-9.el8_10.x86_64", - "Installed: mesa-libEGL-23.1.4-4.el8_10.x86_64", - "Installed: libimobiledevice-1.2.0-16.el8.x86_64", - "Installed: system-config-printer-libs-1.5.11-13.el8.noarch", - "Installed: libinput-1.16.3-3.el8_6.x86_64", - "Installed: mesa-libGL-23.1.4-4.el8_10.x86_64", - "Installed: freetype-devel-2.9.1-10.el8_10.x86_64", - "Installed: mesa-libGL-devel-23.1.4-4.el8_10.x86_64", - "Installed: enchant2-2.2.3-3.el8.x86_64", - "Installed: taglib-1.11.1-8.el8.x86_64", - "Installed: fuse-2.9.7-19.el8.x86_64", - "Installed: pulseaudio-14.0-4.el8.x86_64", - "Installed: iio-sensor-proxy-2.4-3.el8.x86_64", - "Installed: pulseaudio-libs-14.0-4.el8.x86_64", - "Installed: rest-0.8.1-2.el8.x86_64", - "Installed: pulseaudio-libs-glib2-14.0-4.el8.x86_64", - "Installed: pulseaudio-module-bluetooth-14.0-4.el8.x86_64", - "Installed: libappstream-glib-0.7.14-3.el8.x86_64", - "Installed: grilo-0.3.6-3.el8.x86_64", - "Installed: pinentry-gtk-1.1.0-2.el8.x86_64", - "Installed: totem-pl-parser-3.26.1-2.el8.x86_64", - "Installed: mesa-libgbm-23.1.4-4.el8_10.x86_64", - "Installed: pipewire-0.3.6-1.el8.x86_64", - "Installed: evolution-data-server-3.28.5-24.el8.x86_64", - "Installed: libXfont2-devel-2.0.3-2.el8.x86_64", - "Installed: evolution-data-server-langpacks-3.28.5-24.el8.noarch", - "Installed: gsm-1.0.17-5.el8.x86_64", - "Installed: evolution-data-server-ui-3.28.5-24.el8.x86_64", - "Installed: mesa-libglapi-23.1.4-4.el8_10.x86_64", - "Installed: pipewire-libs-0.3.6-1.el8.x86_64", - "Installed: rocky-backgrounds-86.3-1.el8.noarch", - "Installed: mesa-libxatracker-23.1.4-4.el8_10.x86_64", - "Installed: pixman-devel-0.38.4-4.el8.x86_64", - "Installed: gstreamer1-1.16.1-2.el8.x86_64", - "Installed: libstemmer-0-10.585svn.el8.x86_64", - "Installed: libvisual-1:0.4.0-25.el8.x86_64", - "Installed: gstreamer1-plugins-bad-free-1.16.1-5.el8_10.x86_64", - "Installed: Thunar-4.16.8-1.el8.x86_64", - "Installed: libxfce4ui-4.16.0-2.el8.x86_64", - "Installed: gstreamer1-plugins-base-1.16.1-5.el8_10.x86_64", - "Installed: exo-4.16.2-1.el8.x86_64", - "Installed: libxfce4util-4.16.0-4.el8.x86_64", - "Installed: twolame-libs-0.3.13-12.el8.x86_64", - "Installed: libvorbis-1:1.3.6-2.el8.x86_64", - "Installed: gjs-1.56.2-6.el8.x86_64", - "Installed: glib-networking-2.56.1-1.1.el8.x86_64", - "Installed: gstreamer1-plugins-good-1.16.1-5.el8_10.x86_64", - "Installed: libvpx-1.7.0-12.el8_10.x86_64", - "Installed: libwacom-1.6-3.el8.x86_64", - "Installed: libwacom-data-1.6-3.el8.noarch", - "Installed: ModemManager-glib-1.20.2-1.el8.x86_64", - "Installed: libwayland-client-1.21.0-1.el8.x86_64", - "Installed: libwayland-cursor-1.21.0-1.el8.x86_64", - "Installed: libwayland-egl-1.21.0-1.el8.x86_64", - "Installed: libwayland-server-1.21.0-1.el8.x86_64", - "Installed: plymouth-graphics-libs-0.9.4-11.20200615git1e36e30.el8.x86_64", - "Installed: glibmm24-2.56.0-2.el8.x86_64", - "Installed: gtk3-3.22.30-12.el8_10.x86_64", - "Installed: plymouth-plugin-label-0.9.4-11.20200615git1e36e30.el8.x86_64", - "Installed: glx-utils-8.4.0-5.20181118git1830dcb.el8.x86_64", - "Installed: xorg-x11-server-devel-1.20.11-26.el8_10.x86_64", - "Installed: plymouth-plugin-two-step-0.9.4-11.20200615git1e36e30.el8.x86_64", - "Installed: xorg-x11-util-macros-1.19.2-1.el8.noarch", - "Installed: libX11-devel-1.6.8-9.el8_10.x86_64", - "Installed: plymouth-system-theme-0.9.4-11.20200615git1e36e30.el8.x86_64", - "Installed: gnome-bluetooth-1:3.34.3-1.el8.x86_64", - "Installed: gtkmm30-3.22.2-3.el8.x86_64", - "Installed: gnome-bluetooth-libs-1:3.34.3-1.el8.x86_64", - "Installed: gtksourceview3-3.24.9-1.el8.x86_64", - "Installed: libX11-xcb-1.6.8-9.el8_10.x86_64", - "Installed: libwnck3-3.24.1-2.el8.x86_64", - "Installed: libXScrnSaver-1.2.3-1.el8.x86_64", - "Installed: plymouth-theme-charge-0.9.4-11.20200615git1e36e30.el8.x86_64", - "Installed: plymouth-theme-spinner-0.9.4-11.20200615git1e36e30.el8.x86_64", - "Installed: rtkit-0.11-19.el8.x86_64", - "Installed: NetworkManager-wifi-1:1.40.16-19.el8_10.x86_64", - "Installed: gnome-control-center-3.28.2-37.el8.x86_64", - "Installed: gnome-control-center-filesystem-3.28.2-37.el8.noarch", - "Installed: libXau-devel-1.0.9-3.el8.x86_64", - "Installed: gnome-desktop3-3.32.2-3.el8.x86_64", - "Installed: libnma-1.8.38-1.el8.x86_64", - "Installed: libnotify-0.7.7-6.el8.x86_64", - "Installed: libxcb-devel-1.13.1-1.el8.x86_64", - "Installed: upower-0.99.7-4.el8_7.x86_64", - "Installed: liboauth-1.0.3-9.el8.x86_64", - "Installed: libxkbcommon-x11-0.9.1-1.el8.x86_64", - "Installed: libxkbfile-1.1.0-1.el8.x86_64", - "Installed: libogg-2:1.3.2-10.el8.x86_64", - "Installed: libxklavier-5.4-11.el8.x86_64", - "Installed: wpa_supplicant-1:2.10-2.el8.x86_64", - "Installed: libXdmcp-1.1.3-1.el8.x86_64", - "Installed: gnome-keyring-3.28.2-2.el8_10.x86_64", - "Installed: gnome-keyring-pam-3.28.2-2.el8_10.x86_64", - "Installed: python3-idna-2.5-7.el8_10.noarch", - "Installed: gnome-menus-3.13.3-12.el8.x86_64", - "Installed: GConf2-3.2.6-22.el8.x86_64", - "Installed: gnome-online-accounts-3.28.2-7.el8.x86_64", - "Installed: libXfont2-2.0.3-2.el8.x86_64", - "Installed: libxshmfence-1.3-2.el8.x86_64", - "Installed: mpg123-libs-1.32.9-1.el8_10.x86_64", - "Installed: gnome-session-3.28.1-21.el8.x86_64", - "Installed: python3-cairo-1.16.3-6.el8.x86_64", - "Installed: gnome-session-wayland-session-3.28.1-21.el8.x86_64", - "Installed: gnome-session-xsession-3.28.1-21.el8.x86_64", - "Installed: gnome-settings-daemon-3.32.0-20.el8.x86_64", - "Installed: libwbclient-4.19.4-9.el8_10.x86_64", - "Installed: gnome-shell-3.32.2-57.el8_10.x86_64", - "Installed: iso-codes-3.79-2.el8.noarch", - "Installed: libpciaccess-devel-0.14-1.el8.x86_64", - "Installed: mtdev-1.1.5-12.el8.x86_64", - "Installed: cups-pk-helper-0.2.6-5.el8.x86_64", - "Installed: openssh-askpass-8.0p1-25.el8_10.x86_64", - "Installed: mutter-3.32.2-73.el8_10.x86_64", - "Installed: python3-cups-1.9.72-21.el8.rocky.x86_64", - "Installed: libXres-1.2.0-4.el8.x86_64", - "Installed: libplist-2.0.0-10.el8.x86_64", - "Installed: opus-1.3-0.4.beta.el8.x86_64", - "Installed: avahi-glib-0.7-27.el8_10.1.x86_64", - "Installed: vino-3.22.0-11.el8.x86_64", - "Installed: harfbuzz-icu-1.7.5-4.el8.x86_64", - "Installed: libXtst-1.2.3-7.el8.x86_64", - "Installed: orc-0.4.28-4.el8_10.x86_64", - "Installed: libXv-1.0.11-7.el8.x86_64", - "Installed: abattis-cantarell-fonts-0.0.25-6.el8.noarch", - "Installed: dbus-x11-1:1.12.8-26.el8.x86_64", - "Installed: dconf-0.28.0-4.el8.x86_64", - "Installed: libXxf86dga-1.1.5-1.el8.x86_64", - "Installed: python3-gobject-3.28.3-2.el8.x86_64" - ] -} - -TASK [base_os : Include Debian-specific tasks] ********************************* -skipping: [localhost] => { - "changed": false, - "false_condition": "ansible_facts.distribution == 'Debian'", - "skip_reason": "Conditional result was False" -} - -TASK [base_os : Ensure requests package is installed] ************************** -ok: [localhost] => { - "changed": false, - "cmd": [ - "/usr/local/ghpc-venv/bin/python3", - "-m", - "pip.__main__", - "install", - "requests" - ], - "name": [ - "requests" - ], - "requirements": null, - "state": "present", - "version": null, - "virtualenv": null -} - -STDOUT: - -Requirement already satisfied: requests in /usr/local/ghpc-venv/lib64/python3.12/site-packages (2.32.4) -Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/ghpc-venv/lib64/python3.12/site-packages (from requests) (3.4.2) -Requirement already satisfied: idna<4,>=2.5 in /usr/local/ghpc-venv/lib64/python3.12/site-packages (from requests) (3.10) -Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/ghpc-venv/lib64/python3.12/site-packages (from requests) (2.5.0) -Requirement already satisfied: certifi>=2017.4.17 in /usr/local/ghpc-venv/lib64/python3.12/site-packages (from requests) (2025.7.14) - - -TASK [base_os : Ensure Python & tooling are installed] ************************* -ok: [localhost] => { - "changed": false, - "rc": 0, - "results": [] -} - -MSG: - -Nothing to do - -TASK [base_os : Provision GPU setup] ******************************************* -skipping: [localhost] => { - "changed": false, - "false_condition": "nvidia_gpu_present.rc == 0", - "skip_reason": "Conditional result was False" -} - -TASK [base_os : Set current role for lock manager completion] ****************** -ok: [localhost] => { - "ansible_facts": { - "current_role": "base_os", - "role_completed": true - }, - "changed": false -} - -TASK [lock_manager : Debug create lock operation] ****************************** -ok: [localhost] => {} - -MSG: - -Creating/updating lock file for role: base_os - -TASK [lock_manager : Check if lock file exists] ******************************** -ok: [localhost] => { - "changed": false, - "stat": { - "atime": 1753370866.5072315, - "attr_flags": "", - "attributes": [], - "block_size": 4096, - "blocks": 8, - "charset": "us-ascii", - "checksum": "358a7f4095fbb49a0eedafd77e97040d1b9b5a26", - "ctime": 1753370866.2832148, - "dev": 2050, - "device_type": 0, - "executable": false, - "exists": true, - "gid": 0, - "gr_name": "root", - "inode": 891734, - "isblk": false, - "ischr": false, - "isdir": false, - "isfifo": false, - "isgid": false, - "islnk": false, - "isreg": true, - "issock": false, - "isuid": false, - "mimetype": "text/plain", - "mode": "0644", - "mtime": 1753370866.137204, - "nlink": 1, - "path": "/opt/vdi-setup/.vdi-lock.yaml", - "pw_name": "root", - "readable": true, - "rgrp": true, - "roth": true, - "rusr": true, - "size": 879, - "uid": 0, - "version": "3120075969", - "wgrp": false, - "woth": false, - "writeable": true, - "wusr": true, - "xgrp": false, - "xoth": false, - "xusr": false - } -} - -TASK [lock_manager : Debug lock file existence] ******************************** -ok: [localhost] => {} - -MSG: - -Lock file exists: True - -TASK [lock_manager : Create initial lock file structure] *********************** -skipping: [localhost] => { - "changed": false, - "false_condition": "not lock_file_stat.stat.exists", - "skip_reason": "Conditional result was False" -} - -TASK [lock_manager : Debug lock file creation] ********************************* -skipping: [localhost] => { - "false_condition": "debug | default(false) and not lock_file_stat.stat.exists" -} - -TASK [lock_manager : Load existing lock file data] ***************************** -ok: [localhost] => { - "ansible_facts": { - "vdi_setup_status": { - "completed_roles": { - "base_os": { - "completed": false - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:27:44Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:27:44Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "ansible_included_var_files": [ - "/opt/vdi-setup/.vdi-lock.yaml" - ], - "changed": false -} - -TASK [lock_manager : Get current lock data] ************************************ -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": false - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:27:44Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:27:44Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Update deployment hash] *********************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": false - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:27:44Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:27:44Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Get current timestamp] ************************************ -changed: [localhost] => { - "changed": true, - "cmd": [ - "date", - "-u", - "+%Y-%m-%dT%H:%M:%SZ" - ], - "delta": "0:00:00.002192", - "end": "2025-07-24 15:29:50.438490", - "rc": 0, - "start": "2025-07-24 15:29:50.436298" -} - -STDOUT: - -2025-07-24T15:29:50Z - -TASK [lock_manager : Update last updated timestamp] **************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": false - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:29:50Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:27:44Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Get current user secrets status] ************************** -ok: [localhost] => { - "ansible_facts": { - "current_user_secrets": { - "last_secret_check": "2025-07-24T15:27:44Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Create user secrets update] ******************************* -ok: [localhost] => { - "ansible_facts": { - "user_secrets_update": { - "last_secret_check": "2025-07-24T15:29:50Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" - } - }, - "changed": false -} - -TASK [lock_manager : Update user secrets hash] ********************************* -ok: [localhost] => { - "ansible_facts": { - "updated_user_secrets": { - "last_secret_check": "2025-07-24T15:29:50Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Apply user secrets update] ******************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": false - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:29:50Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:29:50Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Create role status entry] ********************************* -ok: [localhost] => { - "ansible_facts": { - "role_status_entry": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - } - }, - "changed": false -} - -TASK [lock_manager : Debug role status entry] ********************************** -ok: [localhost] => {} - -MSG: - -Role status entry for base_os: - - completed: True - - completed_at: 2025-07-24T15:29:50Z - - -TASK [lock_manager : Debug current lock data before role update] *************** -ok: [localhost] => {} - -MSG: - -Current lock data before updating base_os: - - current_lock_data: completed_roles: - base_os: - completed: false - lock_manager: - completed: true - completed_at: '2025-07-24T15:27:44Z' - secret_manager: - completed: false - user_provision: - completed: false - vdi_tool: - completed: false - vnc: - completed: false -created_at: '2025-07-24T15:27:41Z' -deployment_hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab -deployment_name: vdi-test-scott -force_rerun: false -last_updated: '2025-07-24T15:29:50Z' -lock_version: '1.0' -setup_status: configuring -user_management: - current_users: - - alice - - bob - previous_users: [] - removed_users: [] -user_secrets_status: - last_secret_check: '2025-07-24T15:29:50Z' - user_secrets_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 - users_updated: - - alice - - bob - - - current_completed_roles: base_os: - completed: false -lock_manager: - completed: true - completed_at: '2025-07-24T15:27:44Z' -secret_manager: - completed: false -user_provision: - completed: false -vdi_tool: - completed: false -vnc: - completed: false - - - role_completed: True - - current_role: base_os - - -TASK [lock_manager : Get current completed roles] ****************************** -ok: [localhost] => { - "ansible_facts": { - "current_completed_roles": { - "base_os": { - "completed": false - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - } - }, - "changed": false -} - -TASK [lock_manager : Update completed roles] *********************************** -ok: [localhost] => { - "ansible_facts": { - "updated_completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - } - }, - "changed": false -} - -TASK [lock_manager : Update lock data with completed roles] ******************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:29:50Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:29:50Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Debug updated lock data after role update] **************** -ok: [localhost] => {} - -MSG: - -Updated lock data after updating base_os: - - current_role: base_os - - role_completed: True - - completed_roles: {'base_os': {'completed': True, 'completed_at': '2025-07-24T15:29:50Z'}, 'lock_manager': {'completed': True, 'completed_at': '2025-07-24T15:27:44Z'}, 'secret_manager': {'completed': False}, 'user_provision': {'completed': False}, 'vdi_tool': {'completed': False}, 'vnc': {'completed': False}} - - -TASK [lock_manager : Get current users updated list] *************************** -ok: [localhost] => { - "ansible_facts": { - "current_users_updated": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Get new usernames list] *********************************** -ok: [localhost] => { - "ansible_facts": { - "new_usernames": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Combine users updated lists] ****************************** -ok: [localhost] => { - "ansible_facts": { - "combined_users_updated": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Debug user deduplication] ********************************* -ok: [localhost] => {} - -MSG: - -User deduplication for base_os: - - current_users_updated: ['alice', 'bob'] - - new_usernames: ['alice', 'bob'] - - combined_users_updated: ['alice', 'bob'] - - -TASK [lock_manager : Get current user secrets for list update] ***************** -ok: [localhost] => { - "ansible_facts": { - "current_user_secrets_for_list": { - "last_secret_check": "2025-07-24T15:29:50Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Update users updated list] ******************************** -ok: [localhost] => { - "ansible_facts": { - "updated_user_secrets_with_list": { - "last_secret_check": "2025-07-24T15:29:50Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Apply users updated list] ********************************* -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:29:50Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:29:50Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Get current user management data] ************************* -ok: [localhost] => { - "ansible_facts": { - "current_user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [], - "removed_users": [] - } - }, - "changed": false -} - -TASK [lock_manager : Get current usernames list] ******************************* -ok: [localhost] => { - "ansible_facts": { - "current_usernames": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Update user management data] ****************************** -ok: [localhost] => { - "ansible_facts": { - "updated_user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - } - }, - "changed": false -} - -TASK [lock_manager : Apply user management update] ***************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:29:50Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:29:50Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Debug user management] ************************************ -ok: [localhost] => {} - -MSG: - -User management for base_os: - - current_users: ['alice', 'bob'] - - previous_users: ['alice', 'bob'] - - removed_users: [] - - -TASK [lock_manager : Check if all roles are completed] ************************* -ok: [localhost] => { - "ansible_facts": { - "all_roles_completed": false - }, - "changed": false -} - -TASK [lock_manager : Update setup status based on completion] ****************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:29:50Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:29:50Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Debug setup status update] ******************************** -ok: [localhost] => {} - -MSG: - -Setup status update: - - all_roles_completed: False - - setup_status: configuring - - completed_roles_count: 2 - - total_roles_count: 6 - - -TASK [lock_manager : Write updated lock file] ********************************** -changed: [localhost] => { - "changed": true, - "checksum": "9dd77a3585697aef08aed09e819bb4819f75aeb3", - "dest": "/opt/vdi-setup/.vdi-lock.yaml", - "gid": 0, - "group": "root", - "md5sum": "31b80e47ec3641c2918f8ff0403d2956", - "mode": "0644", - "owner": "root", - "size": 940, - "src": "/root/.ansible/tmp/ansible-tmp-1753370991.526421-18369-193730364017589/source", - "state": "file", - "uid": 0 -} - -TASK [lock_manager : Debug VM metadata update] ********************************* -ok: [localhost] => {} - -MSG: - -VM metadata update info: - - hostname: vdi-test-scott-rocky-0 - - zone: us-central1-a - - lock_file_changed: True - - -TASK [lock_manager : Get lock file content as base64] ************************** -changed: [localhost] => { - "changed": true, - "cmd": "cat /opt/vdi-setup/.vdi-lock.yaml | base64 -w 0", - "delta": "0:00:00.003801", - "end": "2025-07-24 15:29:52.075433", - "rc": 0, - "start": "2025-07-24 15:29:52.071632" -} - -STDOUT: - -dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IGZhbHNlCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiBmYWxzZQogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogZmFsc2UKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiBmYWxzZQogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogY29uZmlndXJpbmcKICB1c2VyX21hbmFnZW1lbnQ6CiAgICBjdXJyZW50X3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHByZXZpb3VzX3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHJlbW92ZWRfdXNlcnM6IFtdCiAgdXNlcl9zZWNyZXRzX3N0YXR1czoKICAgIGxhc3Rfc2VjcmV0X2NoZWNrOiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICB1c2VyX3NlY3JldHNfaGFzaDogNmE2YTAyZDNkMDk5MzNjYTI4MmFlMmIwMjMzNjM0M2M5ZDczNzAxZTQ1YmQ4MTQ3NWZiZDkyZGNmOWQyZjNiNgogICAgdXNlcnNfdXBkYXRlZDoKICAgIC0gYWxpY2UKICAgIC0gYm9iCg== - -TASK [lock_manager : Add VDI lock file content to VM metadata] ***************** -changed: [localhost] => { - "changed": true, - "cmd": [ - "gcloud", - "compute", - "instances", - "add-metadata", - "vdi-test-scott-rocky-0", - "--metadata", - "vdi-lock-content=dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IGZhbHNlCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiBmYWxzZQogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogZmFsc2UKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiBmYWxzZQogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogY29uZmlndXJpbmcKICB1c2VyX21hbmFnZW1lbnQ6CiAgICBjdXJyZW50X3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHByZXZpb3VzX3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHJlbW92ZWRfdXNlcnM6IFtdCiAgdXNlcl9zZWNyZXRzX3N0YXR1czoKICAgIGxhc3Rfc2VjcmV0X2NoZWNrOiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICB1c2VyX3NlY3JldHNfaGFzaDogNmE2YTAyZDNkMDk5MzNjYTI4MmFlMmIwMjMzNjM0M2M5ZDczNzAxZTQ1YmQ4MTQ3NWZiZDkyZGNmOWQyZjNiNgogICAgdXNlcnNfdXBkYXRlZDoKICAgIC0gYWxpY2UKICAgIC0gYm9iCg==", - "--zone=us-central1-a" - ], - "delta": "0:00:04.336413", - "end": "2025-07-24 15:29:56.603337", - "failed_when_result": false, - "rc": 0, - "start": "2025-07-24 15:29:52.266924" -} - -STDERR: - -Updated [https://www.googleapis.com/compute/v1/projects/hpc-discovery-external/zones/us-central1-a/instances/vdi-test-scott-rocky-0]. - -TASK [lock_manager : Debug lock file write] ************************************ -ok: [localhost] => {} - -MSG: - -Lock file written: True - -TASK [lock_manager : Verify lock file integrity] ******************************* -ok: [localhost] => { - "changed": false, - "cmd": [ - "python3", - "-c", - "import yaml; yaml.safe_load(open('/opt/vdi-setup/.vdi-lock.yaml'))" - ], - "delta": "0:00:00.039517", - "end": "2025-07-24 15:29:56.877395", - "rc": 0, - "start": "2025-07-24 15:29:56.837878" -} - -TASK [lock_manager : Debug lock file validation] ******************************* -ok: [localhost] => {} - -MSG: - -Lock file validation: True - -TASK [lock_manager : Check if VDI monitor service exists] ********************** -ok: [localhost] => { - "changed": false, - "stat": { - "atime": 1753370874.1548011, - "attr_flags": "", - "attributes": [], - "block_size": 4096, - "blocks": 8, - "charset": "us-ascii", - "checksum": "f6de22707f8d9248014f679a2f1930451e2bf635", - "ctime": 1753370872.6256874, - "dev": 2050, - "device_type": 0, - "executable": false, - "exists": true, - "gid": 0, - "gr_name": "root", - "inode": 285214140, - "isblk": false, - "ischr": false, - "isdir": false, - "isfifo": false, - "isgid": false, - "islnk": false, - "isreg": true, - "issock": false, - "isuid": false, - "mimetype": "text/plain", - "mode": "0644", - "mtime": 1753370872.4766762, - "nlink": 1, - "path": "/etc/systemd/system/vdi-monitor.service", - "pw_name": "root", - "readable": true, - "rgrp": true, - "roth": true, - "rusr": true, - "size": 283, - "uid": 0, - "version": "1338188335", - "wgrp": false, - "woth": false, - "writeable": true, - "wusr": true, - "xgrp": false, - "xoth": false, - "xusr": false - } -} - -TASK [lock_manager : Debug VDI monitor setup conditions] *********************** -ok: [localhost] => {} - -MSG: - -VDI monitor setup conditions: - - current_role: base_os - - vdi_monitor_service_exists: True - - lock_file_creation.changed: False - - will_setup_monitor: False - - -TASK [lock_manager : Setup VDI monitoring service (first time only)] *********** -skipping: [localhost] => { - "changed": false, - "false_condition": "not vdi_monitor_service_stat.stat.exists", - "skip_reason": "Conditional result was False" -} - -TASK [lock_manager : Fail if lock file is invalid] ***************************** -skipping: [localhost] => { - "changed": false, - "false_condition": "lock_file_validation.rc != 0", - "skip_reason": "Conditional result was False" -} - -TASK [secret_manager : Set current role for lock manager check] **************** -ok: [localhost] => { - "ansible_facts": { - "current_role": "secret_manager" - }, - "changed": false -} - -TASK [lock_manager : Check if lock file exists] ******************************** -ok: [localhost] => { - "changed": false, - "stat": { - "atime": 1753370992.073591, - "attr_flags": "", - "attributes": [], - "block_size": 4096, - "blocks": 8, - "charset": "us-ascii", - "checksum": "9dd77a3585697aef08aed09e819bb4819f75aeb3", - "ctime": 1753370991.845574, - "dev": 2050, - "device_type": 0, - "executable": false, - "exists": true, - "gid": 0, - "gr_name": "root", - "inode": 185115083, - "isblk": false, - "ischr": false, - "isdir": false, - "isfifo": false, - "isgid": false, - "islnk": false, - "isreg": true, - "issock": false, - "isuid": false, - "mimetype": "text/plain", - "mode": "0644", - "mtime": 1753370991.6915624, - "nlink": 1, - "path": "/opt/vdi-setup/.vdi-lock.yaml", - "pw_name": "root", - "readable": true, - "rgrp": true, - "roth": true, - "rusr": true, - "size": 940, - "uid": 0, - "version": "3443573269", - "wgrp": false, - "woth": false, - "writeable": true, - "wusr": true, - "xgrp": false, - "xoth": false, - "xusr": false - } -} - -TASK [lock_manager : Debug lock file existence] ******************************** -ok: [localhost] => {} - -MSG: - -Lock file exists: True - -TASK [lock_manager : Load existing lock file] ********************************** -ok: [localhost] => { - "ansible_facts": { - "vdi_setup_status": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:29:50Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:29:50Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "ansible_included_var_files": [ - "/opt/vdi-setup/.vdi-lock.yaml" - ], - "changed": false -} - -TASK [lock_manager : Debug lock file load result] ****************************** -ok: [localhost] => {} - -MSG: - -Lock file loaded successfully: True - -TASK [lock_manager : Calculate current deployment hash] ************************ -ok: [localhost] => { - "ansible_facts": { - "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab" - }, - "changed": false -} - -TASK [lock_manager : Debug deployment hash calculation] ************************ -ok: [localhost] => {} - -MSG: - -Current deployment hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab -Deployment components: - - deployment_name: vdi-test-scott - - project_id: hpc-discovery-external - - vdi_tool: guacamole - - vnc_flavor: tigervnc - - user_provision: local_users - - vdi_webapp_port: 8081 - - vdi_resolution: 1920x1080 - - vdi_users count: 2 - - -TASK [lock_manager : Calculate current user secrets hash] ********************** -ok: [localhost] => { - "ansible_facts": { - "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" - }, - "changed": false -} - -TASK [lock_manager : Debug user secrets hash calculation] ********************** -ok: [localhost] => {} - -MSG: - -Current user secrets hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 -User configuration components: - - vdi_users (blueprint): [{"username": "alice", "port": 5901}, {"username": "bob", "port": 5902, "secret_name": "a-password-for-bob"}] - - -TASK [lock_manager : Set default values for missing lock file] ***************** -skipping: [localhost] => { - "changed": false, - "false_condition": "not lock_file_stat.stat.exists", - "skip_reason": "Conditional result was False" -} - -TASK [lock_manager : Debug lock file status] *********************************** -ok: [localhost] => {} - -MSG: - -Lock file status: - - exists: True - - deployment_hash: none - - user_secrets_hash: none - - force_rerun: False - - -TASK [lock_manager : Check if lock file exists] ******************************** -ok: [localhost] => { - "ansible_facts": { - "lock_file_exists": true - }, - "changed": false -} - -TASK [lock_manager : Check if current role is completed] *********************** -ok: [localhost] => { - "ansible_facts": { - "current_role_completed": false - }, - "changed": false -} - -TASK [lock_manager : Check if deployment hash matches] ************************* -ok: [localhost] => { - "ansible_facts": { - "deployment_hash_matches": false - }, - "changed": false -} - -TASK [lock_manager : Check if force rerun is enabled] ************************** -ok: [localhost] => { - "ansible_facts": { - "force_rerun_enabled": false - }, - "changed": false -} - -TASK [lock_manager : Determine if role should run] ***************************** -ok: [localhost] => { - "ansible_facts": { - "role_should_run": true - }, - "changed": false -} - -TASK [lock_manager : Debug role should run calculation] ************************ -ok: [localhost] => {} - -MSG: - -Role should run calculation for secret_manager: - - not lock_file_exists: False - - not current_role_completed: True - - not deployment_hash_matches: True - - force_rerun_enabled: False - - role_should_run: True - - -TASK [lock_manager : Debug role execution decision] **************************** -ok: [localhost] => {} - -MSG: - -Role execution decision for secret_manager: - - current_role: secret_manager - - ansible_role_name: lock_manager - - role_should_run: True - - lock_file_exists: True - - current_role_completed: False - - deployment_hash_matches: False - - force_rerun_enabled: False - - -TASK [lock_manager : Check if user configuration has changed] ****************** -ok: [localhost] => { - "ansible_facts": { - "user_config_changed": true - }, - "changed": false -} - -TASK [lock_manager : Set user secrets changed flag (will be updated by secret_manager role)] *** -ok: [localhost] => { - "ansible_facts": { - "user_secrets_changed": true - }, - "changed": false -} - -TASK [lock_manager : Debug user secrets change detection] ********************** -ok: [localhost] => {} - -MSG: - -User configuration change detection: - - user_config_changed: True - - user_secrets_changed: True - - stored_hash: none - - current_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 - - -TASK [lock_manager : Set lock check result] ************************************ -ok: [localhost] => { - "ansible_facts": { - "lock_check_result": { - "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "existing_deployment_hash": "none", - "existing_user_secrets_hash": "none", - "force_rerun": false, - "role_should_run": true, - "user_secrets_changed": true - } - }, - "changed": false -} - -TASK [lock_manager : Debug final lock check result] **************************** -ok: [localhost] => {} - -MSG: - -Final lock check result: - - role_should_run: True - - deployment_hash_changed: True - - user_secrets_changed: True - - force_rerun: False - - -TASK [secret_manager : Set role_should_run from lock check result] ************* -ok: [localhost] => { - "ansible_facts": { - "role_should_run": true - }, - "changed": false -} - -TASK [secret_manager : Skip secret_manager tasks if role should not run] ******* -skipping: [localhost] => { - "false_condition": "not role_should_run" -} - -TASK [secret_manager : Generate random database password] ********************** -ok: [localhost] => { - "ansible_facts": { - "database_password": "H95L0yeLji7pzjDCOuemJDzEwoASP7Nq" - }, - "changed": false -} - -TASK [secret_manager : Generate random webapp admin password] ****************** -ok: [localhost] => { - "ansible_facts": { - "webapp_admin_password": "n7d53VAcYV4xT86sDI8mNKO7BHsIIfa0" - }, - "changed": false -} - -TASK [secret_manager : Compute Webapp admin password hash to save to Webapp server] *** -ok: [localhost] => { - "ansible_facts": { - "webapp_admin_hash": "E4440760F8847690907CADBB2B87C3E21015556DB4DAEA4A0D9C5D5F85DD6EA9" - }, - "changed": false -} - -TASK [secret_manager : Get access token for Secret Manager API] **************** -ok: [localhost] => { - "changed": false, - "connection": "Close", - "content": "{\"access_token\":\"ya29.c.c0ASRK0Gb5OH5v-xtWlMorXSZswjZqBApOSzy3U4RHvOuuIn3lLmwmFBxGPcpUUw72yV0h0ow9Xuo-4544iIzBnhfADz7iDQ09v99UJ3y8RTi6yqwoXjOBcbI1Xqkj1gPMgVsUBGPfooSczQEcQPj27MBL6rLrDadQnacLMKVzogosGtpxH3Kf-a5lztK5AmcyYrgGxfyFqtPbF34J9J-88UghA5vbVdYScappzlkaxWbwoAmK2xW0Z40kE2uz91P2wwgcG1JUAcR6QgOc2_NarPeRkZ1_kOl8COE6l05gNix4hcH5j8PBTsaoUHre-vE-j6gAJRIe2uMUlVa7Mm3CkuJqrfxR3hNXUpsu6mdsTJEDvJz02U4uIMwYrupU58mzflgo1q0al4hW0UEVWwE411P2y-gSmWQY71t8ynep0kpl-alZF3zxk-mhFkfurxl06mj9I9pq7cYb9-Zu3B-S7qv5pmg6ISQQVk33QgXO15n913orUWhFf2qlVtOqMhzgsgF12iB6rwXtts4y1ceodU6JaqkWf3wsF0uQarJwbb7eYg5jBzVjcxwcuy2Y7bhOqfIa0X88nW7_WgqUXXe7FBIQYfJoJ0U1aZjS3Frwicnjr4gMBoW9OQuQcX-ygjUXBmWMXxWpmW5tOnB1xcQxey-jbeypuys1ou0JFIql8Q4iIJyQvWBuchJXaWsyfooRJfr_mXSpf6f2fm0dW1kV3abBx_jIisS7UnUq8hnZyieo515yZ1_ktq-1gicxlum3SltkiSbmYxnJ12vaM3W-t74bi1JRZV2jJess0vFssRrapFzBoQzfUUbuR21hh1ImdmQ6_YlhUh9IJcX-qkk_X24nXuQfdSUptj0WXx14cWs6cfQ_b5-qfnFyymtlcMuS3i7mg3z1vghaf2-24J5IUUYxxOaqJ7_J48pVzaXpJnypnymuU_lp-w88jk_I0zYtUiSuFq5SrsY9_aIQqtOY40pyZdmYO2zg5w4fBpzShjhfpXMI8BicoUz\",\"expires_in\":3599,\"token_type\":\"Bearer\"}", - "content_length": "1083", - "content_type": "application/json", - "cookies": {}, - "cookies_string": "", - "date": "Thu, 24 Jul 2025 15:29:58 GMT", - "elapsed": 0, - "json": { - "access_token": "ya29.c.c0ASRK0Gb5OH5v-xtWlMorXSZswjZqBApOSzy3U4RHvOuuIn3lLmwmFBxGPcpUUw72yV0h0ow9Xuo-4544iIzBnhfADz7iDQ09v99UJ3y8RTi6yqwoXjOBcbI1Xqkj1gPMgVsUBGPfooSczQEcQPj27MBL6rLrDadQnacLMKVzogosGtpxH3Kf-a5lztK5AmcyYrgGxfyFqtPbF34J9J-88UghA5vbVdYScappzlkaxWbwoAmK2xW0Z40kE2uz91P2wwgcG1JUAcR6QgOc2_NarPeRkZ1_kOl8COE6l05gNix4hcH5j8PBTsaoUHre-vE-j6gAJRIe2uMUlVa7Mm3CkuJqrfxR3hNXUpsu6mdsTJEDvJz02U4uIMwYrupU58mzflgo1q0al4hW0UEVWwE411P2y-gSmWQY71t8ynep0kpl-alZF3zxk-mhFkfurxl06mj9I9pq7cYb9-Zu3B-S7qv5pmg6ISQQVk33QgXO15n913orUWhFf2qlVtOqMhzgsgF12iB6rwXtts4y1ceodU6JaqkWf3wsF0uQarJwbb7eYg5jBzVjcxwcuy2Y7bhOqfIa0X88nW7_WgqUXXe7FBIQYfJoJ0U1aZjS3Frwicnjr4gMBoW9OQuQcX-ygjUXBmWMXxWpmW5tOnB1xcQxey-jbeypuys1ou0JFIql8Q4iIJyQvWBuchJXaWsyfooRJfr_mXSpf6f2fm0dW1kV3abBx_jIisS7UnUq8hnZyieo515yZ1_ktq-1gicxlum3SltkiSbmYxnJ12vaM3W-t74bi1JRZV2jJess0vFssRrapFzBoQzfUUbuR21hh1ImdmQ6_YlhUh9IJcX-qkk_X24nXuQfdSUptj0WXx14cWs6cfQ_b5-qfnFyymtlcMuS3i7mg3z1vghaf2-24J5IUUYxxOaqJ7_J48pVzaXpJnypnymuU_lp-w88jk_I0zYtUiSuFq5SrsY9_aIQqtOY40pyZdmYO2zg5w4fBpzShjhfpXMI8BicoUz", - "expires_in": 3599, - "token_type": "Bearer" - }, - "metadata_flavor": "Google", - "redirected": false, - "server": "Metadata Server for VM", - "status": 200, - "url": "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token", - "x_frame_options": "SAMEORIGIN", - "x_xss_protection": "0" -} - -MSG: - -OK (1083 bytes) - -TASK [secret_manager : Create a GCP secret for webapp server login] ************ -ok: [localhost] => { - "accept_ranges": "none", - "changed": false, - "connection": "close", - "content_type": "application/json; charset=UTF-8", - "date": "Thu, 24 Jul 2025 15:29:58 GMT", - "elapsed": 0, - "json": { - "error": { - "code": 409, - "message": "Secret [projects/748006908949/secrets/webapp-server-password-vdi-test-scott] already exists.", - "status": "ALREADY_EXISTS" - } - }, - "redirected": false, - "server": "ESF", - "status": 409, - "transfer_encoding": "chunked", - "url": "https://secretmanager.googleapis.com/v1/projects/hpc-discovery-external/secrets?secretId=webapp-server-password-vdi-test-scott", - "vary": "Origin,Accept-Encoding", - "x_content_type_options": "nosniff", - "x_frame_options": "SAMEORIGIN", - "x_xss_protection": "0" -} - -MSG: - -HTTP Error 409: Conflict - -TASK [secret_manager : Add webapp password version to secret] ****************** -ok: [localhost] => { - "accept_ranges": "none", - "changed": false, - "connection": "close", - "content_type": "application/json; charset=UTF-8", - "cookies": {}, - "cookies_string": "", - "date": "Thu, 24 Jul 2025 15:29:59 GMT", - "elapsed": 0, - "json": { - "createTime": "2025-07-24T15:29:59.522597Z", - "etag": "\"163aae7f960525\"", - "name": "projects/748006908949/secrets/webapp-server-password-vdi-test-scott/versions/82", - "replicationStatus": { - "automatic": {} - }, - "state": "ENABLED" - }, - "redirected": false, - "server": "ESF", - "status": 200, - "transfer_encoding": "chunked", - "url": "https://secretmanager.googleapis.com/v1/projects/hpc-discovery-external/secrets/webapp-server-password-vdi-test-scott:addVersion", - "vary": "X-Origin, Referer, Origin,Accept-Encoding", - "x_content_type_options": "nosniff", - "x_frame_options": "SAMEORIGIN", - "x_xss_protection": "0" -} - -MSG: - -OK (unknown bytes) - -TASK [secret_manager : Create a GCP secret for database password] ************** -ok: [localhost] => { - "accept_ranges": "none", - "changed": false, - "connection": "close", - "content_type": "application/json; charset=UTF-8", - "date": "Thu, 24 Jul 2025 15:29:59 GMT", - "elapsed": 0, - "json": { - "error": { - "code": 409, - "message": "Secret [projects/748006908949/secrets/db-password-vdi-test-scott] already exists.", - "status": "ALREADY_EXISTS" - } - }, - "redirected": false, - "server": "ESF", - "status": 409, - "transfer_encoding": "chunked", - "url": "https://secretmanager.googleapis.com/v1/projects/hpc-discovery-external/secrets?secretId=db-password-vdi-test-scott", - "vary": "Origin,Accept-Encoding", - "x_content_type_options": "nosniff", - "x_frame_options": "SAMEORIGIN", - "x_xss_protection": "0" -} - -MSG: - -HTTP Error 409: Conflict - -TASK [secret_manager : Add database password version to secret] **************** -ok: [localhost] => { - "accept_ranges": "none", - "changed": false, - "connection": "close", - "content_type": "application/json; charset=UTF-8", - "cookies": {}, - "cookies_string": "", - "date": "Thu, 24 Jul 2025 15:30:00 GMT", - "elapsed": 0, - "json": { - "createTime": "2025-07-24T15:30:00.398932Z", - "etag": "\"163aae7fa36454\"", - "name": "projects/748006908949/secrets/db-password-vdi-test-scott/versions/82", - "replicationStatus": { - "automatic": {} - }, - "state": "ENABLED" - }, - "redirected": false, - "server": "ESF", - "status": 200, - "transfer_encoding": "chunked", - "url": "https://secretmanager.googleapis.com/v1/projects/hpc-discovery-external/secrets/db-password-vdi-test-scott:addVersion", - "vary": "X-Origin, Referer, Origin,Accept-Encoding", - "x_content_type_options": "nosniff", - "x_frame_options": "SAMEORIGIN", - "x_xss_protection": "0" -} - -MSG: - -OK (unknown bytes) - -TASK [secret_manager : Generate per-user secrets and enrich list] ************** -included: /tmp/vdi/roles/secret_manager/tasks/user_secret_tasks.yaml for localhost => (item={'username': 'alice', 'port': 5901}) -included: /tmp/vdi/roles/secret_manager/tasks/user_secret_tasks.yaml for localhost => (item={'username': 'bob', 'port': 5902, 'secret_name': 'a-password-for-bob'}) - -TASK [secret_manager : Compute default secret_name if none provided] *********** -ok: [localhost] => { - "ansible_facts": { - "secret_name": "vdi-user-password-alice-vdi-test-scott" - }, - "changed": false -} - -TASK [secret_manager : Determine password source for user alice] *************** -ok: [localhost] => { - "ansible_facts": { - "password_source": "generate" - }, - "changed": false -} - -TASK [secret_manager : Clean password source value] **************************** -ok: [localhost] => { - "ansible_facts": { - "password_source": "generate" - }, - "changed": false -} - -TASK [secret_manager : Debug password source for alice] ************************ -ok: [localhost] => {} - -MSG: - -Password source for alice: - - password_source: "generate" - - secret_name: not set - - password: not set - - -TASK [secret_manager : Fetch existing Secret Manager password for user alice] *** -skipping: [localhost] => { - "changed": false, - "false_condition": "password_source == \"secret\"", - "skip_reason": "Conditional result was False" -} - -TASK [secret_manager : Debug Secret Manager fetch result for alice] ************ -skipping: [localhost] => { - "false_condition": "debug | default(false) and password_source == \"secret\"" -} - -TASK [secret_manager : Set VDI user password from Secret Manager] ************** -skipping: [localhost] => { - "changed": false, - "false_condition": "password_source == \"secret\" and sm_result.status == 200", - "skip_reason": "Conditional result was False" -} - -TASK [secret_manager : Check if password has actually changed for alice] ******* -skipping: [localhost] => { - "changed": false, - "false_condition": "password_source == \"secret\" and sm_result.status == 200", - "skip_reason": "Conditional result was False" -} - -TASK [secret_manager : Debug password change detection for alice] ************** -skipping: [localhost] => { - "false_condition": "debug | default(false) and password_source == \"secret\"" -} - -TASK [secret_manager : Generate random password for user alice] **************** -ok: [localhost] => { - "ansible_facts": { - "vdiuser_password": "xc61q4jU3mUCo84M" - }, - "changed": false -} - -TASK [secret_manager : Set user password from provided value] ****************** -skipping: [localhost] => { - "changed": false, - "false_condition": "password_source == \"password\"", - "skip_reason": "Conditional result was False" -} - -TASK [secret_manager : Create secret for user alice (provided password)] ******* -skipping: [localhost] => { - "changed": false, - "false_condition": "password_source == \"password\"", - "skip_reason": "Conditional result was False" -} - -TASK [secret_manager : Store provided password in Secret Manager for user alice] *** -skipping: [localhost] => { - "changed": false, - "false_condition": "password_source == \"password\"", - "skip_reason": "Conditional result was False" -} - -TASK [secret_manager : Create secret for user alice] *************************** -ok: [localhost] => { - "accept_ranges": "none", - "changed": false, - "connection": "close", - "content_type": "application/json; charset=UTF-8", - "date": "Thu, 24 Jul 2025 15:30:01 GMT", - "elapsed": 0, - "json": { - "error": { - "code": 409, - "message": "Secret [projects/748006908949/secrets/vdi-user-password-alice-vdi-test-scott] already exists.", - "status": "ALREADY_EXISTS" - } - }, - "redirected": false, - "server": "ESF", - "status": 409, - "transfer_encoding": "chunked", - "url": "https://secretmanager.googleapis.com/v1/projects/hpc-discovery-external/secrets?secretId=vdi-user-password-alice-vdi-test-scott", - "vary": "Origin,Accept-Encoding", - "x_content_type_options": "nosniff", - "x_frame_options": "SAMEORIGIN", - "x_xss_protection": "0" -} - -MSG: - -HTTP Error 409: Conflict - -TASK [secret_manager : Store generated password in Secret Manager for user alice] *** -ok: [localhost] => { - "accept_ranges": "none", - "changed": false, - "connection": "close", - "content_type": "application/json; charset=UTF-8", - "cookies": {}, - "cookies_string": "", - "date": "Thu, 24 Jul 2025 15:30:01 GMT", - "elapsed": 0, - "json": { - "createTime": "2025-07-24T15:30:01.698173Z", - "etag": "\"163aae7fb7377d\"", - "name": "projects/748006908949/secrets/vdi-user-password-alice-vdi-test-scott/versions/63", - "replicationStatus": { - "automatic": {} - }, - "state": "ENABLED" - }, - "redirected": false, - "server": "ESF", - "status": 200, - "transfer_encoding": "chunked", - "url": "https://secretmanager.googleapis.com/v1/projects/hpc-discovery-external/secrets/vdi-user-password-alice-vdi-test-scott:addVersion", - "vary": "X-Origin, Referer, Origin,Accept-Encoding", - "x_content_type_options": "nosniff", - "x_frame_options": "SAMEORIGIN", - "x_xss_protection": "0" -} - -MSG: - -OK (unknown bytes) - -TASK [secret_manager : Set VNC configuration for user] ************************* -ok: [localhost] => { - "ansible_facts": { - "vnc_config": { - "display_number": 1, - "vncserver_password": "4JB1YHUQ" - } - }, - "changed": false -} - -TASK [secret_manager : Build enriched user list] ******************************* -ok: [localhost] => { - "ansible_facts": { - "vdi_users_updated": [ - { - "display_number": 1, - "password": "xc61q4jU3mUCo84M", - "port": 5901, - "username": "alice", - "vncserver_password": "4JB1YHUQ" - } - ] - }, - "changed": false -} - -TASK [secret_manager : Compute default secret_name if none provided] *********** -skipping: [localhost] => { - "changed": false, - "false_condition": "item.secret_name is not defined", - "skip_reason": "Conditional result was False" -} - -TASK [secret_manager : Determine password source for user bob] ***************** -ok: [localhost] => { - "ansible_facts": { - "password_source": "secret" - }, - "changed": false -} - -TASK [secret_manager : Clean password source value] **************************** -ok: [localhost] => { - "ansible_facts": { - "password_source": "secret" - }, - "changed": false -} - -TASK [secret_manager : Debug password source for bob] ************************** -ok: [localhost] => {} - -MSG: - -Password source for bob: - - password_source: "secret" - - secret_name: a-password-for-bob - - password: not set - - -TASK [secret_manager : Fetch existing Secret Manager password for user bob] **** -ok: [localhost] => { - "accept_ranges": "none", - "changed": false, - "connection": "close", - "content": "{\n \"name\": \"projects/748006908949/secrets/a-password-for-bob/versions/4\",\n \"payload\": {\n \"data\": \"QjBicDRzc3cwcmQxMjM=\",\n \"dataCrc32c\": \"3219829350\"\n }\n}\n", - "content_type": "application/json; charset=UTF-8", - "cookies": {}, - "cookies_string": "", - "date": "Thu, 24 Jul 2025 15:30:02 GMT", - "elapsed": 0, - "failed_when_result": false, - "json": { - "name": "projects/748006908949/secrets/a-password-for-bob/versions/4", - "payload": { - "data": "QjBicDRzc3cwcmQxMjM=", - "dataCrc32c": "3219829350" - } - }, - "redirected": false, - "server": "ESF", - "status": 200, - "transfer_encoding": "chunked", - "url": "https://secretmanager.googleapis.com/v1/projects/hpc-discovery-external/secrets/a-password-for-bob/versions/latest:access", - "vary": "X-Origin, Referer, Origin,Accept-Encoding", - "x_content_type_options": "nosniff", - "x_frame_options": "SAMEORIGIN", - "x_xss_protection": "0" -} - -MSG: - -OK (unknown bytes) - -TASK [secret_manager : Debug Secret Manager fetch result for bob] ************** -ok: [localhost] => {} - -MSG: - -Secret Manager fetch for bob: - - status: 200 - - success: True - - user_config_changed: True - - password_source: secret - - -TASK [secret_manager : Set VDI user password from Secret Manager] ************** -ok: [localhost] => { - "ansible_facts": { - "vdiuser_password": "B0bp4ssw0rd123" - }, - "changed": false -} - -TASK [secret_manager : Check if password has actually changed for bob] ********* -ok: [localhost] => { - "ansible_facts": { - "password_actually_changed": true - }, - "changed": false -} - -TASK [secret_manager : Debug password change detection for bob] **************** -ok: [localhost] => {} - -MSG: - -Password change detection for bob: - - user_config_changed: True - - password_source: secret - - sm_result_status: 200 - - password_actually_changed: True - - -TASK [secret_manager : Generate random password for user bob] ****************** -skipping: [localhost] => { - "changed": false, - "false_condition": "password_source == \"generate\"", - "skip_reason": "Conditional result was False" -} - -TASK [secret_manager : Set user password from provided value] ****************** -skipping: [localhost] => { - "changed": false, - "false_condition": "password_source == \"password\"", - "skip_reason": "Conditional result was False" -} - -TASK [secret_manager : Create secret for user bob (provided password)] ********* -skipping: [localhost] => { - "changed": false, - "false_condition": "password_source == \"password\"", - "skip_reason": "Conditional result was False" -} - -TASK [secret_manager : Store provided password in Secret Manager for user bob] *** -skipping: [localhost] => { - "changed": false, - "false_condition": "password_source == \"password\"", - "skip_reason": "Conditional result was False" -} - -TASK [secret_manager : Create secret for user bob] ***************************** -skipping: [localhost] => { - "changed": false, - "false_condition": "password_source == \"generate\"", - "skip_reason": "Conditional result was False" -} - -TASK [secret_manager : Store generated password in Secret Manager for user bob] *** -skipping: [localhost] => { - "changed": false, - "false_condition": "password_source == \"generate\"", - "skip_reason": "Conditional result was False" -} - -TASK [secret_manager : Set VNC configuration for user] ************************* -ok: [localhost] => { - "ansible_facts": { - "vnc_config": { - "display_number": 2, - "vncserver_password": "7DmxvFp2" - } - }, - "changed": false -} - -TASK [secret_manager : Build enriched user list] ******************************* -ok: [localhost] => { - "ansible_facts": { - "vdi_users_updated": [ - { - "display_number": 1, - "password": "xc61q4jU3mUCo84M", - "port": 5901, - "username": "alice", - "vncserver_password": "4JB1YHUQ" - }, - { - "display_number": 2, - "password": "B0bp4ssw0rd123", - "port": 5902, - "secret_name": "a-password-for-bob", - "username": "bob", - "vncserver_password": "7DmxvFp2" - } - ] - }, - "changed": false -} - -TASK [secret_manager : Set enriched vdi_users_updated variable] **************** -ok: [localhost] => { - "ansible_facts": { - "vdi_users_updated": [ - { - "display_number": 1, - "password": "xc61q4jU3mUCo84M", - "port": 5901, - "username": "alice", - "vncserver_password": "4JB1YHUQ" - }, - { - "display_number": 2, - "password": "B0bp4ssw0rd123", - "port": 5902, - "secret_name": "a-password-for-bob", - "username": "bob", - "vncserver_password": "7DmxvFp2" - } - ] - }, - "changed": false -} - -TASK [secret_manager : Set current role for lock manager completion] *********** -ok: [localhost] => { - "ansible_facts": { - "current_role": "secret_manager", - "role_completed": true - }, - "changed": false -} - -TASK [lock_manager : Debug create lock operation] ****************************** -ok: [localhost] => {} - -MSG: - -Creating/updating lock file for role: secret_manager - -TASK [lock_manager : Check if lock file exists] ******************************** -ok: [localhost] => { - "changed": false, - "stat": { - "atime": 1753370992.073591, - "attr_flags": "", - "attributes": [], - "block_size": 4096, - "blocks": 8, - "charset": "us-ascii", - "checksum": "9dd77a3585697aef08aed09e819bb4819f75aeb3", - "ctime": 1753370991.845574, - "dev": 2050, - "device_type": 0, - "executable": false, - "exists": true, - "gid": 0, - "gr_name": "root", - "inode": 185115083, - "isblk": false, - "ischr": false, - "isdir": false, - "isfifo": false, - "isgid": false, - "islnk": false, - "isreg": true, - "issock": false, - "isuid": false, - "mimetype": "text/plain", - "mode": "0644", - "mtime": 1753370991.6915624, - "nlink": 1, - "path": "/opt/vdi-setup/.vdi-lock.yaml", - "pw_name": "root", - "readable": true, - "rgrp": true, - "roth": true, - "rusr": true, - "size": 940, - "uid": 0, - "version": "3443573269", - "wgrp": false, - "woth": false, - "writeable": true, - "wusr": true, - "xgrp": false, - "xoth": false, - "xusr": false - } -} - -TASK [lock_manager : Debug lock file existence] ******************************** -ok: [localhost] => {} - -MSG: - -Lock file exists: True - -TASK [lock_manager : Create initial lock file structure] *********************** -skipping: [localhost] => { - "changed": false, - "false_condition": "not lock_file_stat.stat.exists", - "skip_reason": "Conditional result was False" -} - -TASK [lock_manager : Debug lock file creation] ********************************* -skipping: [localhost] => { - "false_condition": "debug | default(false) and not lock_file_stat.stat.exists" -} - -TASK [lock_manager : Load existing lock file data] ***************************** -ok: [localhost] => { - "ansible_facts": { - "vdi_setup_status": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:29:50Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:29:50Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "ansible_included_var_files": [ - "/opt/vdi-setup/.vdi-lock.yaml" - ], - "changed": false -} - -TASK [lock_manager : Get current lock data] ************************************ -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:29:50Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:29:50Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Update deployment hash] *********************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:29:50Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:29:50Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Get current timestamp] ************************************ -changed: [localhost] => { - "changed": true, - "cmd": [ - "date", - "-u", - "+%Y-%m-%dT%H:%M:%SZ" - ], - "delta": "0:00:00.002549", - "end": "2025-07-24 15:30:03.439894", - "rc": 0, - "start": "2025-07-24 15:30:03.437345" -} - -STDOUT: - -2025-07-24T15:30:03Z - -TASK [lock_manager : Update last updated timestamp] **************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:03Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:29:50Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Get current user secrets status] ************************** -ok: [localhost] => { - "ansible_facts": { - "current_user_secrets": { - "last_secret_check": "2025-07-24T15:29:50Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Create user secrets update] ******************************* -ok: [localhost] => { - "ansible_facts": { - "user_secrets_update": { - "last_secret_check": "2025-07-24T15:30:03Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" - } - }, - "changed": false -} - -TASK [lock_manager : Update user secrets hash] ********************************* -ok: [localhost] => { - "ansible_facts": { - "updated_user_secrets": { - "last_secret_check": "2025-07-24T15:30:03Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Apply user secrets update] ******************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:03Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:03Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Create role status entry] ********************************* -ok: [localhost] => { - "ansible_facts": { - "role_status_entry": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - } - }, - "changed": false -} - -TASK [lock_manager : Debug role status entry] ********************************** -ok: [localhost] => {} - -MSG: - -Role status entry for secret_manager: - - completed: True - - completed_at: 2025-07-24T15:30:03Z - - -TASK [lock_manager : Debug current lock data before role update] *************** -ok: [localhost] => {} - -MSG: - -Current lock data before updating secret_manager: - - current_lock_data: completed_roles: - base_os: - completed: true - completed_at: '2025-07-24T15:29:50Z' - lock_manager: - completed: true - completed_at: '2025-07-24T15:27:44Z' - secret_manager: - completed: false - user_provision: - completed: false - vdi_tool: - completed: false - vnc: - completed: false -created_at: '2025-07-24T15:27:41Z' -deployment_hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab -deployment_name: vdi-test-scott -force_rerun: false -last_updated: '2025-07-24T15:30:03Z' -lock_version: '1.0' -setup_status: configuring -user_management: - current_users: - - alice - - bob - previous_users: - - alice - - bob - removed_users: [] -user_secrets_status: - last_secret_check: '2025-07-24T15:30:03Z' - user_secrets_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 - users_updated: - - alice - - bob - - - current_completed_roles: base_os: - completed: true - completed_at: '2025-07-24T15:29:50Z' -lock_manager: - completed: true - completed_at: '2025-07-24T15:27:44Z' -secret_manager: - completed: false -user_provision: - completed: false -vdi_tool: - completed: false -vnc: - completed: false - - - role_completed: True - - current_role: secret_manager - - -TASK [lock_manager : Get current completed roles] ****************************** -ok: [localhost] => { - "ansible_facts": { - "current_completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": false - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - } - }, - "changed": false -} - -TASK [lock_manager : Update completed roles] *********************************** -ok: [localhost] => { - "ansible_facts": { - "updated_completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - } - }, - "changed": false -} - -TASK [lock_manager : Update lock data with completed roles] ******************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:03Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:03Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Debug updated lock data after role update] **************** -ok: [localhost] => {} - -MSG: - -Updated lock data after updating secret_manager: - - current_role: secret_manager - - role_completed: True - - completed_roles: {'base_os': {'completed': True, 'completed_at': '2025-07-24T15:29:50Z'}, 'lock_manager': {'completed': True, 'completed_at': '2025-07-24T15:27:44Z'}, 'secret_manager': {'completed': True, 'completed_at': '2025-07-24T15:30:03Z'}, 'user_provision': {'completed': False}, 'vdi_tool': {'completed': False}, 'vnc': {'completed': False}} - - -TASK [lock_manager : Get current users updated list] *************************** -ok: [localhost] => { - "ansible_facts": { - "current_users_updated": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Get new usernames list] *********************************** -ok: [localhost] => { - "ansible_facts": { - "new_usernames": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Combine users updated lists] ****************************** -ok: [localhost] => { - "ansible_facts": { - "combined_users_updated": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Debug user deduplication] ********************************* -ok: [localhost] => {} - -MSG: - -User deduplication for secret_manager: - - current_users_updated: ['alice', 'bob'] - - new_usernames: ['alice', 'bob'] - - combined_users_updated: ['alice', 'bob'] - - -TASK [lock_manager : Get current user secrets for list update] ***************** -ok: [localhost] => { - "ansible_facts": { - "current_user_secrets_for_list": { - "last_secret_check": "2025-07-24T15:30:03Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Update users updated list] ******************************** -ok: [localhost] => { - "ansible_facts": { - "updated_user_secrets_with_list": { - "last_secret_check": "2025-07-24T15:30:03Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Apply users updated list] ********************************* -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:03Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:03Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Get current user management data] ************************* -ok: [localhost] => { - "ansible_facts": { - "current_user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - } - }, - "changed": false -} - -TASK [lock_manager : Get current usernames list] ******************************* -ok: [localhost] => { - "ansible_facts": { - "current_usernames": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Update user management data] ****************************** -ok: [localhost] => { - "ansible_facts": { - "updated_user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - } - }, - "changed": false -} - -TASK [lock_manager : Apply user management update] ***************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:03Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:03Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Debug user management] ************************************ -ok: [localhost] => {} - -MSG: - -User management for secret_manager: - - current_users: ['alice', 'bob'] - - previous_users: ['alice', 'bob'] - - removed_users: [] - - -TASK [lock_manager : Check if all roles are completed] ************************* -ok: [localhost] => { - "ansible_facts": { - "all_roles_completed": false - }, - "changed": false -} - -TASK [lock_manager : Update setup status based on completion] ****************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:03Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:03Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Debug setup status update] ******************************** -ok: [localhost] => {} - -MSG: - -Setup status update: - - all_roles_completed: False - - setup_status: configuring - - completed_roles_count: 3 - - total_roles_count: 6 - - -TASK [lock_manager : Write updated lock file] ********************************** -changed: [localhost] => { - "changed": true, - "checksum": "9205f17d8ce64bf55a046c433f6f24c4efa12eae", - "dest": "/opt/vdi-setup/.vdi-lock.yaml", - "gid": 0, - "group": "root", - "md5sum": "8a1ac8aa138bd51d4b3fd6f087ff52cb", - "mode": "0644", - "owner": "root", - "size": 982, - "src": "/root/.ansible/tmp/ansible-tmp-1753371004.5096657-25387-190441536237874/source", - "state": "file", - "uid": 0 -} - -TASK [lock_manager : Debug VM metadata update] ********************************* -ok: [localhost] => {} - -MSG: - -VM metadata update info: - - hostname: vdi-test-scott-rocky-0 - - zone: us-central1-a - - lock_file_changed: True - - -TASK [lock_manager : Get lock file content as base64] ************************** -changed: [localhost] => { - "changed": true, - "cmd": "cat /opt/vdi-setup/.vdi-lock.yaml | base64 -w 0", - "delta": "0:00:00.003695", - "end": "2025-07-24 15:30:05.054073", - "rc": 0, - "start": "2025-07-24 15:30:05.050378" -} - -STDOUT: - -dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiBmYWxzZQogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogZmFsc2UKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiBmYWxzZQogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogY29uZmlndXJpbmcKICB1c2VyX21hbmFnZW1lbnQ6CiAgICBjdXJyZW50X3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHByZXZpb3VzX3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHJlbW92ZWRfdXNlcnM6IFtdCiAgdXNlcl9zZWNyZXRzX3N0YXR1czoKICAgIGxhc3Rfc2VjcmV0X2NoZWNrOiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgICB1c2VyX3NlY3JldHNfaGFzaDogNmE2YTAyZDNkMDk5MzNjYTI4MmFlMmIwMjMzNjM0M2M5ZDczNzAxZTQ1YmQ4MTQ3NWZiZDkyZGNmOWQyZjNiNgogICAgdXNlcnNfdXBkYXRlZDoKICAgIC0gYWxpY2UKICAgIC0gYm9iCg== - -TASK [lock_manager : Add VDI lock file content to VM metadata] ***************** -changed: [localhost] => { - "changed": true, - "cmd": [ - "gcloud", - "compute", - "instances", - "add-metadata", - "vdi-test-scott-rocky-0", - "--metadata", - "vdi-lock-content=dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiBmYWxzZQogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogZmFsc2UKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiBmYWxzZQogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogY29uZmlndXJpbmcKICB1c2VyX21hbmFnZW1lbnQ6CiAgICBjdXJyZW50X3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHByZXZpb3VzX3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHJlbW92ZWRfdXNlcnM6IFtdCiAgdXNlcl9zZWNyZXRzX3N0YXR1czoKICAgIGxhc3Rfc2VjcmV0X2NoZWNrOiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgICB1c2VyX3NlY3JldHNfaGFzaDogNmE2YTAyZDNkMDk5MzNjYTI4MmFlMmIwMjMzNjM0M2M5ZDczNzAxZTQ1YmQ4MTQ3NWZiZDkyZGNmOWQyZjNiNgogICAgdXNlcnNfdXBkYXRlZDoKICAgIC0gYWxpY2UKICAgIC0gYm9iCg==", - "--zone=us-central1-a" - ], - "delta": "0:00:04.452715", - "end": "2025-07-24 15:30:09.703135", - "failed_when_result": false, - "rc": 0, - "start": "2025-07-24 15:30:05.250420" -} - -STDERR: - -Updated [https://www.googleapis.com/compute/v1/projects/hpc-discovery-external/zones/us-central1-a/instances/vdi-test-scott-rocky-0]. - -TASK [lock_manager : Debug lock file write] ************************************ -ok: [localhost] => {} - -MSG: - -Lock file written: True - -TASK [lock_manager : Verify lock file integrity] ******************************* -ok: [localhost] => { - "changed": false, - "cmd": [ - "python3", - "-c", - "import yaml; yaml.safe_load(open('/opt/vdi-setup/.vdi-lock.yaml'))" - ], - "delta": "0:00:00.039944", - "end": "2025-07-24 15:30:09.991625", - "rc": 0, - "start": "2025-07-24 15:30:09.951681" -} - -TASK [lock_manager : Debug lock file validation] ******************************* -ok: [localhost] => {} - -MSG: - -Lock file validation: True - -TASK [lock_manager : Check if VDI monitor service exists] ********************** -ok: [localhost] => { - "changed": false, - "stat": { - "atime": 1753370874.1548011, - "attr_flags": "", - "attributes": [], - "block_size": 4096, - "blocks": 8, - "charset": "us-ascii", - "checksum": "f6de22707f8d9248014f679a2f1930451e2bf635", - "ctime": 1753370872.6256874, - "dev": 2050, - "device_type": 0, - "executable": false, - "exists": true, - "gid": 0, - "gr_name": "root", - "inode": 285214140, - "isblk": false, - "ischr": false, - "isdir": false, - "isfifo": false, - "isgid": false, - "islnk": false, - "isreg": true, - "issock": false, - "isuid": false, - "mimetype": "text/plain", - "mode": "0644", - "mtime": 1753370872.4766762, - "nlink": 1, - "path": "/etc/systemd/system/vdi-monitor.service", - "pw_name": "root", - "readable": true, - "rgrp": true, - "roth": true, - "rusr": true, - "size": 283, - "uid": 0, - "version": "1338188335", - "wgrp": false, - "woth": false, - "writeable": true, - "wusr": true, - "xgrp": false, - "xoth": false, - "xusr": false - } -} - -TASK [lock_manager : Debug VDI monitor setup conditions] *********************** -ok: [localhost] => {} - -MSG: - -VDI monitor setup conditions: - - current_role: secret_manager - - vdi_monitor_service_exists: True - - lock_file_creation.changed: False - - will_setup_monitor: False - - -TASK [lock_manager : Setup VDI monitoring service (first time only)] *********** -skipping: [localhost] => { - "changed": false, - "false_condition": "not vdi_monitor_service_stat.stat.exists", - "skip_reason": "Conditional result was False" -} - -TASK [lock_manager : Fail if lock file is invalid] ***************************** -skipping: [localhost] => { - "changed": false, - "false_condition": "lock_file_validation.rc != 0", - "skip_reason": "Conditional result was False" -} - -TASK [user_provision : Set current role for lock manager check] **************** -ok: [localhost] => { - "ansible_facts": { - "current_role": "user_provision" - }, - "changed": false -} - -TASK [lock_manager : Check if lock file exists] ******************************** -ok: [localhost] => { - "changed": false, - "stat": { - "atime": 1753371005.05256, - "attr_flags": "", - "attributes": [], - "block_size": 4096, - "blocks": 8, - "charset": "us-ascii", - "checksum": "9205f17d8ce64bf55a046c433f6f24c4efa12eae", - "ctime": 1753371004.8265433, - "dev": 2050, - "device_type": 0, - "executable": false, - "exists": true, - "gid": 0, - "gr_name": "root", - "inode": 51539232, - "isblk": false, - "ischr": false, - "isdir": false, - "isfifo": false, - "isgid": false, - "islnk": false, - "isreg": true, - "issock": false, - "isuid": false, - "mimetype": "text/plain", - "mode": "0644", - "mtime": 1753371004.6825325, - "nlink": 1, - "path": "/opt/vdi-setup/.vdi-lock.yaml", - "pw_name": "root", - "readable": true, - "rgrp": true, - "roth": true, - "rusr": true, - "size": 982, - "uid": 0, - "version": "2720298527", - "wgrp": false, - "woth": false, - "writeable": true, - "wusr": true, - "xgrp": false, - "xoth": false, - "xusr": false - } -} - -TASK [lock_manager : Debug lock file existence] ******************************** -ok: [localhost] => {} - -MSG: - -Lock file exists: True - -TASK [lock_manager : Load existing lock file] ********************************** -ok: [localhost] => { - "ansible_facts": { - "vdi_setup_status": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:03Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:03Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "ansible_included_var_files": [ - "/opt/vdi-setup/.vdi-lock.yaml" - ], - "changed": false -} - -TASK [lock_manager : Debug lock file load result] ****************************** -ok: [localhost] => {} - -MSG: - -Lock file loaded successfully: True - -TASK [lock_manager : Calculate current deployment hash] ************************ -ok: [localhost] => { - "ansible_facts": { - "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab" - }, - "changed": false -} - -TASK [lock_manager : Debug deployment hash calculation] ************************ -ok: [localhost] => {} - -MSG: - -Current deployment hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab -Deployment components: - - deployment_name: vdi-test-scott - - project_id: hpc-discovery-external - - vdi_tool: guacamole - - vnc_flavor: tigervnc - - user_provision: local_users - - vdi_webapp_port: 8081 - - vdi_resolution: 1920x1080 - - vdi_users count: 2 - - -TASK [lock_manager : Calculate current user secrets hash] ********************** -ok: [localhost] => { - "ansible_facts": { - "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" - }, - "changed": false -} - -TASK [lock_manager : Debug user secrets hash calculation] ********************** -ok: [localhost] => {} - -MSG: - -Current user secrets hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 -User configuration components: - - vdi_users (blueprint): [{"username": "alice", "port": 5901}, {"username": "bob", "port": 5902, "secret_name": "a-password-for-bob"}] - - -TASK [lock_manager : Set default values for missing lock file] ***************** -skipping: [localhost] => { - "changed": false, - "false_condition": "not lock_file_stat.stat.exists", - "skip_reason": "Conditional result was False" -} - -TASK [lock_manager : Debug lock file status] *********************************** -ok: [localhost] => {} - -MSG: - -Lock file status: - - exists: True - - deployment_hash: none - - user_secrets_hash: none - - force_rerun: False - - -TASK [lock_manager : Check if lock file exists] ******************************** -ok: [localhost] => { - "ansible_facts": { - "lock_file_exists": true - }, - "changed": false -} - -TASK [lock_manager : Check if current role is completed] *********************** -ok: [localhost] => { - "ansible_facts": { - "current_role_completed": false - }, - "changed": false -} - -TASK [lock_manager : Check if deployment hash matches] ************************* -ok: [localhost] => { - "ansible_facts": { - "deployment_hash_matches": false - }, - "changed": false -} - -TASK [lock_manager : Check if force rerun is enabled] ************************** -ok: [localhost] => { - "ansible_facts": { - "force_rerun_enabled": false - }, - "changed": false -} - -TASK [lock_manager : Determine if role should run] ***************************** -ok: [localhost] => { - "ansible_facts": { - "role_should_run": true - }, - "changed": false -} - -TASK [lock_manager : Debug role should run calculation] ************************ -ok: [localhost] => {} - -MSG: - -Role should run calculation for user_provision: - - not lock_file_exists: False - - not current_role_completed: True - - not deployment_hash_matches: True - - force_rerun_enabled: False - - role_should_run: True - - -TASK [lock_manager : Debug role execution decision] **************************** -ok: [localhost] => {} - -MSG: - -Role execution decision for user_provision: - - current_role: user_provision - - ansible_role_name: lock_manager - - role_should_run: True - - lock_file_exists: True - - current_role_completed: False - - deployment_hash_matches: False - - force_rerun_enabled: False - - -TASK [lock_manager : Check if user configuration has changed] ****************** -ok: [localhost] => { - "ansible_facts": { - "user_config_changed": true - }, - "changed": false -} - -TASK [lock_manager : Set user secrets changed flag (will be updated by secret_manager role)] *** -ok: [localhost] => { - "ansible_facts": { - "user_secrets_changed": true - }, - "changed": false -} - -TASK [lock_manager : Debug user secrets change detection] ********************** -ok: [localhost] => {} - -MSG: - -User configuration change detection: - - user_config_changed: True - - user_secrets_changed: True - - stored_hash: none - - current_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 - - -TASK [lock_manager : Set lock check result] ************************************ -ok: [localhost] => { - "ansible_facts": { - "lock_check_result": { - "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "existing_deployment_hash": "none", - "existing_user_secrets_hash": "none", - "force_rerun": false, - "role_should_run": true, - "user_secrets_changed": true - } - }, - "changed": false -} - -TASK [lock_manager : Debug final lock check result] **************************** -ok: [localhost] => {} - -MSG: - -Final lock check result: - - role_should_run: True - - deployment_hash_changed: True - - user_secrets_changed: True - - force_rerun: False - - -TASK [user_provision : Set role_should_run from lock check result] ************* -ok: [localhost] => { - "ansible_facts": { - "role_should_run": true - }, - "changed": false -} - -TASK [user_provision : Skip user_provision tasks if role should not run] ******* -skipping: [localhost] => { - "false_condition": "not role_should_run" -} - -TASK [user_provision : Ensure 'wheel' group is present] ************************ -ok: [localhost] => { - "changed": false, - "gid": 10, - "name": "wheel", - "state": "present", - "system": false -} - -TASK [user_provision : Ensure VDI user group exists] *************************** -changed: [localhost] => { - "changed": true, - "gid": 1001, - "name": "vdiusers", - "state": "present", - "system": false -} - -TASK [user_provision : Provision local users] ********************************** -included: /tmp/vdi/roles/user_provision/tasks/local_users.yaml for localhost => (item={'username': 'alice', 'port': 5901, 'password': 'xc61q4jU3mUCo84M', 'vncserver_password': '4JB1YHUQ', 'display_number': 1}) -included: /tmp/vdi/roles/user_provision/tasks/local_users.yaml for localhost => (item={'username': 'bob', 'port': 5902, 'secret_name': 'a-password-for-bob', 'password': 'B0bp4ssw0rd123', 'vncserver_password': '7DmxvFp2', 'display_number': 2}) - -TASK [user_provision : Check if user exists] *********************************** -fatal: [localhost]: FAILED! => { - "changed": false -} - -MSG: - -One or more supplied key could not be found in the database. -...ignoring - -TASK [user_provision : Set user exists status for debug] *********************** -ok: [localhost] => { - "ansible_facts": { - "user_exists_status": false - }, - "changed": false -} - -TASK [user_provision : Set user exists status for debug (success case)] ******** -skipping: [localhost] => { - "changed": false, - "false_condition": "debug | default(false) and user_exists_check is not failed", - "skip_reason": "Conditional result was False" -} - -TASK [user_provision : Debug user provision for alice] ************************* -ok: [localhost] => {} - -MSG: - -Provisioning user alice: - - user_secrets_changed: True - - password_actually_changed: True - - password_update_mode: always - - user_exists: False - - -TASK [user_provision : Create VDI user alice] ********************************** -[DEPRECATION WARNING]: Encryption using the Python crypt module is deprecated. -The Python crypt module is deprecated and will be removed from Python 3.13. -Install the passlib library for continued encryption functionality. This -feature will be removed in version 2.17. Deprecation warnings can be disabled -by setting deprecation_warnings=False in ansible.cfg. -changed: [localhost] => { - "changed": true, - "comment": "", - "create_home": true, - "group": 1002, - "groups": "vdiusers,wheel", - "home": "/home/alice", - "name": "alice", - "password": "NOT_LOGGING_PASSWORD", - "shell": "/bin/bash", - "state": "present", - "system": false, - "uid": 1000 -} - -TASK [user_provision : Ensure .ssh directory for alice] ************************ -changed: [localhost] => { - "changed": true, - "gid": 1001, - "group": "vdiusers", - "mode": "0700", - "owner": "alice", - "path": "/home/alice/.ssh", - "size": 6, - "state": "directory", - "uid": 1000 -} - -TASK [user_provision : Generate SSH key pair for alice] ************************ -changed: [localhost] => { - "changed": true, - "comment": "", - "filename": "/home/alice/.ssh/id_rsa", - "fingerprint": "SHA256:4q3JaXL5hMpzved5z0zkLeFXpxKwhFioOkCgEo9i81A", - "public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCxte3GQvt8uFeofNaIUfEMGldCrg9DN8/rUTHzq/dicUI3WuaQKWMC+22TAd7xMernwPlsCUfl3+o5RI1g8vwOGyXmF8dlXKbcokBL2976Ek+7jsibyqzcGiBRKPpN2Rc37L6VMaXupQ+kor/d0zSJXfLVewaWoQcOCktTVFwek7mpVHDbwBAFHN9ruR4M7ItDaJg6fOpD3/cgaegz+L0IIATbXXYGxYTQXnRMEvydcsDFWY8bXKON+YBtZhglc8Y7DJadM2J3S7ZxGwCU2dA4dTRYDTHIeLno0CbWul++qnyRW3OHj0DtdYqtrwi/CnstwYmJ5E9vPctdcGJokw1CYKomgrlpernuRytxrlt2SA6UHyX9VfSiBSuRDkMsyLF3Y/m/nYiDwDjp68XStRt6TmkZNb6zzJnKcRjKmaNiRi5Q1fBULcW4+UWC17SoMX4a74+6WMAtWwDVZvC+UwiCHEJRCKmAMFEE1EDMSsBLX223qXEBIDaXGoyErbezNr3ZUUPR5ZABIfhQW/LwGMW1+WE+k/pFoKeMicHc0EpdNfcZoHEqHKPoKsZl0U8FWJVjLWQO8jVQlUHLhapzA/6vU/JHnpRZ1PrUH6v0PNPjgDhoaaOo7HfdlVKJFNuPV1jC5re1lmBHnZLzyLr8dojaAjElnTr8RRKRHE2X8KTg4w==", - "size": 4096, - "type": "rsa" -} - -TASK [user_provision : Add public key to authorized_keys for alice] ************ -changed: [localhost] => { - "changed": true, - "comment": null, - "exclusive": false, - "follow": false, - "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCxte3GQvt8uFeofNaIUfEMGldCrg9DN8/rUTHzq/dicUI3WuaQKWMC+22TAd7xMernwPlsCUfl3+o5RI1g8vwOGyXmF8dlXKbcokBL2976Ek+7jsibyqzcGiBRKPpN2Rc37L6VMaXupQ+kor/d0zSJXfLVewaWoQcOCktTVFwek7mpVHDbwBAFHN9ruR4M7ItDaJg6fOpD3/cgaegz+L0IIATbXXYGxYTQXnRMEvydcsDFWY8bXKON+YBtZhglc8Y7DJadM2J3S7ZxGwCU2dA4dTRYDTHIeLno0CbWul++qnyRW3OHj0DtdYqtrwi/CnstwYmJ5E9vPctdcGJokw1CYKomgrlpernuRytxrlt2SA6UHyX9VfSiBSuRDkMsyLF3Y/m/nYiDwDjp68XStRt6TmkZNb6zzJnKcRjKmaNiRi5Q1fBULcW4+UWC17SoMX4a74+6WMAtWwDVZvC+UwiCHEJRCKmAMFEE1EDMSsBLX223qXEBIDaXGoyErbezNr3ZUUPR5ZABIfhQW/LwGMW1+WE+k/pFoKeMicHc0EpdNfcZoHEqHKPoKsZl0U8FWJVjLWQO8jVQlUHLhapzA/6vU/JHnpRZ1PrUH6v0PNPjgDhoaaOo7HfdlVKJFNuPV1jC5re1lmBHnZLzyLr8dojaAjElnTr8RRKRHE2X8KTg4w==", - "key_options": null, - "keyfile": "/home/alice/.ssh/authorized_keys", - "manage_dir": true, - "path": null, - "state": "present", - "user": "alice", - "validate_certs": true -} - -TASK [user_provision : Get users to remove from lock file] ********************* -ok: [localhost] => { - "ansible_facts": { - "users_to_remove": [] - }, - "changed": false -} - -TASK [user_provision : Debug local user cleanup] ******************************* -ok: [localhost] => {} - -MSG: - -Local user cleanup: - - users_to_remove: [] - - -TASK [user_provision : Remove local user accounts] ***************************** -skipping: [localhost] => { - "changed": false, - "skipped_reason": "No items in the list" -} - -TASK [user_provision : Check if user exists] *********************************** -fatal: [localhost]: FAILED! => { - "changed": false -} - -MSG: - -One or more supplied key could not be found in the database. -...ignoring - -TASK [user_provision : Set user exists status for debug] *********************** -ok: [localhost] => { - "ansible_facts": { - "user_exists_status": false - }, - "changed": false -} - -TASK [user_provision : Set user exists status for debug (success case)] ******** -skipping: [localhost] => { - "changed": false, - "false_condition": "debug | default(false) and user_exists_check is not failed", - "skip_reason": "Conditional result was False" -} - -TASK [user_provision : Debug user provision for bob] *************************** -ok: [localhost] => {} - -MSG: - -Provisioning user bob: - - user_secrets_changed: True - - password_actually_changed: True - - password_update_mode: always - - user_exists: False - - -TASK [user_provision : Create VDI user bob] ************************************ -[DEPRECATION WARNING]: Encryption using the Python crypt module is deprecated. -The Python crypt module is deprecated and will be removed from Python 3.13. -Install the passlib library for continued encryption functionality. This -feature will be removed in version 2.17. Deprecation warnings can be disabled -by setting deprecation_warnings=False in ansible.cfg. -changed: [localhost] => { - "changed": true, - "comment": "", - "create_home": true, - "group": 1003, - "groups": "vdiusers,wheel", - "home": "/home/bob", - "name": "bob", - "password": "NOT_LOGGING_PASSWORD", - "shell": "/bin/bash", - "state": "present", - "system": false, - "uid": 1001 -} - -TASK [user_provision : Ensure .ssh directory for bob] ************************** -changed: [localhost] => { - "changed": true, - "gid": 1001, - "group": "vdiusers", - "mode": "0700", - "owner": "bob", - "path": "/home/bob/.ssh", - "size": 6, - "state": "directory", - "uid": 1001 -} - -TASK [user_provision : Generate SSH key pair for bob] ************************** -changed: [localhost] => { - "changed": true, - "comment": "", - "filename": "/home/bob/.ssh/id_rsa", - "fingerprint": "SHA256:lKDSzJMXof3DNSJMGYZaAIQO6AFMq0pI5o2to6VG4ic", - "public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCzDZtYeeQAFPzpE14NfR0OHKtADWTNEfulc02pYEGcm+fBptIrFd4IH4J8CF26rUGcdryQ+blRc4UtyUyrIZurK9Q+I8kIM0ggxnLmCM6rs2ycKAayJQihQI8wIa8mQr2OYea4g44fdZr79tZSpGkCmeT9zE432nPDtBxfXw9/zOm+d086lejm/CWxRFidGGiiodWLJR3Sg0VqChij9+TKfplAZCtfuNcKJGywEVkpvNBZ3webAmqDKwgN1Q5cM6H4DntC2FT9jBx6IqCOmXQzQSVjO5BGU1djBLWL/ANJQ9pTfiPxLee4Jl1rT89UhnnN4dJMCrKkEVz4Sf82MUU+S2mnfnlk6U3NSXPfsruy0puttwN0FPjWeq5f5vNCCwuI9PBbmbS4++QyqSgE5JN+C46SlOnxyMQxM0bp5VTmB8AyjXiN5JrTUHlf+H0JIAqspHMEAVaD8jiM0CZ4Oh7Z/t71ZGIMWvY/93IKk5uVEizpazBS42TgmbprjGMTbTLKi3w8oAAQulNR3pnfyt4XekFbmVXlKK12A1nmQgaGx2naUm1bmtUT4yiRb1VDHKHS8+SGjLkKzgud/tkodFMUUhbCNlTgRjhJ78VIkYCIDyymNANbd2cB4wk9tpw845mau47Lvw6tUl33Anh97HUCl5CUI/lcLgOAXpasdD1JnQ==", - "size": 4096, - "type": "rsa" -} - -TASK [user_provision : Add public key to authorized_keys for bob] ************** -changed: [localhost] => { - "changed": true, - "comment": null, - "exclusive": false, - "follow": false, - "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCzDZtYeeQAFPzpE14NfR0OHKtADWTNEfulc02pYEGcm+fBptIrFd4IH4J8CF26rUGcdryQ+blRc4UtyUyrIZurK9Q+I8kIM0ggxnLmCM6rs2ycKAayJQihQI8wIa8mQr2OYea4g44fdZr79tZSpGkCmeT9zE432nPDtBxfXw9/zOm+d086lejm/CWxRFidGGiiodWLJR3Sg0VqChij9+TKfplAZCtfuNcKJGywEVkpvNBZ3webAmqDKwgN1Q5cM6H4DntC2FT9jBx6IqCOmXQzQSVjO5BGU1djBLWL/ANJQ9pTfiPxLee4Jl1rT89UhnnN4dJMCrKkEVz4Sf82MUU+S2mnfnlk6U3NSXPfsruy0puttwN0FPjWeq5f5vNCCwuI9PBbmbS4++QyqSgE5JN+C46SlOnxyMQxM0bp5VTmB8AyjXiN5JrTUHlf+H0JIAqspHMEAVaD8jiM0CZ4Oh7Z/t71ZGIMWvY/93IKk5uVEizpazBS42TgmbprjGMTbTLKi3w8oAAQulNR3pnfyt4XekFbmVXlKK12A1nmQgaGx2naUm1bmtUT4yiRb1VDHKHS8+SGjLkKzgud/tkodFMUUhbCNlTgRjhJ78VIkYCIDyymNANbd2cB4wk9tpw845mau47Lvw6tUl33Anh97HUCl5CUI/lcLgOAXpasdD1JnQ==", - "key_options": null, - "keyfile": "/home/bob/.ssh/authorized_keys", - "manage_dir": true, - "path": null, - "state": "present", - "user": "bob", - "validate_certs": true -} - -TASK [user_provision : Get users to remove from lock file] ********************* -ok: [localhost] => { - "ansible_facts": { - "users_to_remove": [] - }, - "changed": false -} - -TASK [user_provision : Debug local user cleanup] ******************************* -ok: [localhost] => {} - -MSG: - -Local user cleanup: - - users_to_remove: [] - - -TASK [user_provision : Remove local user accounts] ***************************** -skipping: [localhost] => { - "changed": false, - "skipped_reason": "No items in the list" -} - -TASK [user_provision : Provision OS Login users] ******************************* -skipping: [localhost] => (item={'username': 'alice', 'port': 5901, 'password': 'xc61q4jU3mUCo84M', 'vncserver_password': '4JB1YHUQ', 'display_number': 1}) => { - "ansible_loop_var": "item", - "changed": false, - "false_condition": "user_provision | lower == 'os_login'", - "item": { - "display_number": 1, - "password": "xc61q4jU3mUCo84M", - "port": 5901, - "username": "alice", - "vncserver_password": "4JB1YHUQ" - }, - "skip_reason": "Conditional result was False" -} -skipping: [localhost] => (item={'username': 'bob', 'port': 5902, 'secret_name': 'a-password-for-bob', 'password': 'B0bp4ssw0rd123', 'vncserver_password': '7DmxvFp2', 'display_number': 2}) => { - "ansible_loop_var": "item", - "changed": false, - "false_condition": "user_provision | lower == 'os_login'", - "item": { - "display_number": 2, - "password": "B0bp4ssw0rd123", - "port": 5902, - "secret_name": "a-password-for-bob", - "username": "bob", - "vncserver_password": "7DmxvFp2" - }, - "skip_reason": "Conditional result was False" -} -skipping: [localhost] => { - "changed": false -} - -MSG: - -All items skipped - -TASK [user_provision : Set current role for lock manager completion] *********** -ok: [localhost] => { - "ansible_facts": { - "current_role": "user_provision", - "role_completed": true - }, - "changed": false -} - -TASK [lock_manager : Debug create lock operation] ****************************** -ok: [localhost] => {} - -MSG: - -Creating/updating lock file for role: user_provision - -TASK [lock_manager : Check if lock file exists] ******************************** -ok: [localhost] => { - "changed": false, - "stat": { - "atime": 1753371005.05256, - "attr_flags": "", - "attributes": [], - "block_size": 4096, - "blocks": 8, - "charset": "us-ascii", - "checksum": "9205f17d8ce64bf55a046c433f6f24c4efa12eae", - "ctime": 1753371004.8265433, - "dev": 2050, - "device_type": 0, - "executable": false, - "exists": true, - "gid": 0, - "gr_name": "root", - "inode": 51539232, - "isblk": false, - "ischr": false, - "isdir": false, - "isfifo": false, - "isgid": false, - "islnk": false, - "isreg": true, - "issock": false, - "isuid": false, - "mimetype": "text/plain", - "mode": "0644", - "mtime": 1753371004.6825325, - "nlink": 1, - "path": "/opt/vdi-setup/.vdi-lock.yaml", - "pw_name": "root", - "readable": true, - "rgrp": true, - "roth": true, - "rusr": true, - "size": 982, - "uid": 0, - "version": "2720298527", - "wgrp": false, - "woth": false, - "writeable": true, - "wusr": true, - "xgrp": false, - "xoth": false, - "xusr": false - } -} - -TASK [lock_manager : Debug lock file existence] ******************************** -ok: [localhost] => {} - -MSG: - -Lock file exists: True - -TASK [lock_manager : Create initial lock file structure] *********************** -skipping: [localhost] => { - "changed": false, - "false_condition": "not lock_file_stat.stat.exists", - "skip_reason": "Conditional result was False" -} - -TASK [lock_manager : Debug lock file creation] ********************************* -skipping: [localhost] => { - "false_condition": "debug | default(false) and not lock_file_stat.stat.exists" -} - -TASK [lock_manager : Load existing lock file data] ***************************** -ok: [localhost] => { - "ansible_facts": { - "vdi_setup_status": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:03Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:03Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "ansible_included_var_files": [ - "/opt/vdi-setup/.vdi-lock.yaml" - ], - "changed": false -} - -TASK [lock_manager : Get current lock data] ************************************ -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:03Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:03Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Update deployment hash] *********************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:03Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:03Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Get current timestamp] ************************************ -changed: [localhost] => { - "changed": true, - "cmd": [ - "date", - "-u", - "+%Y-%m-%dT%H:%M:%SZ" - ], - "delta": "0:00:00.002381", - "end": "2025-07-24 15:30:16.755905", - "rc": 0, - "start": "2025-07-24 15:30:16.753524" -} - -STDOUT: - -2025-07-24T15:30:16Z - -TASK [lock_manager : Update last updated timestamp] **************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:16Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:03Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Get current user secrets status] ************************** -ok: [localhost] => { - "ansible_facts": { - "current_user_secrets": { - "last_secret_check": "2025-07-24T15:30:03Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Create user secrets update] ******************************* -ok: [localhost] => { - "ansible_facts": { - "user_secrets_update": { - "last_secret_check": "2025-07-24T15:30:16Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" - } - }, - "changed": false -} - -TASK [lock_manager : Update user secrets hash] ********************************* -ok: [localhost] => { - "ansible_facts": { - "updated_user_secrets": { - "last_secret_check": "2025-07-24T15:30:16Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Apply user secrets update] ******************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:16Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:16Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Create role status entry] ********************************* -ok: [localhost] => { - "ansible_facts": { - "role_status_entry": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - } - }, - "changed": false -} - -TASK [lock_manager : Debug role status entry] ********************************** -ok: [localhost] => {} - -MSG: - -Role status entry for user_provision: - - completed: True - - completed_at: 2025-07-24T15:30:16Z - - -TASK [lock_manager : Debug current lock data before role update] *************** -ok: [localhost] => {} - -MSG: - -Current lock data before updating user_provision: - - current_lock_data: completed_roles: - base_os: - completed: true - completed_at: '2025-07-24T15:29:50Z' - lock_manager: - completed: true - completed_at: '2025-07-24T15:27:44Z' - secret_manager: - completed: true - completed_at: '2025-07-24T15:30:03Z' - user_provision: - completed: false - vdi_tool: - completed: false - vnc: - completed: false -created_at: '2025-07-24T15:27:41Z' -deployment_hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab -deployment_name: vdi-test-scott -force_rerun: false -last_updated: '2025-07-24T15:30:16Z' -lock_version: '1.0' -setup_status: configuring -user_management: - current_users: - - alice - - bob - previous_users: - - alice - - bob - removed_users: [] -user_secrets_status: - last_secret_check: '2025-07-24T15:30:16Z' - user_secrets_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 - users_updated: - - alice - - bob - - - current_completed_roles: base_os: - completed: true - completed_at: '2025-07-24T15:29:50Z' -lock_manager: - completed: true - completed_at: '2025-07-24T15:27:44Z' -secret_manager: - completed: true - completed_at: '2025-07-24T15:30:03Z' -user_provision: - completed: false -vdi_tool: - completed: false -vnc: - completed: false - - - role_completed: True - - current_role: user_provision - - -TASK [lock_manager : Get current completed roles] ****************************** -ok: [localhost] => { - "ansible_facts": { - "current_completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": false - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - } - }, - "changed": false -} - -TASK [lock_manager : Update completed roles] *********************************** -ok: [localhost] => { - "ansible_facts": { - "updated_completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - } - }, - "changed": false -} - -TASK [lock_manager : Update lock data with completed roles] ******************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:16Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:16Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Debug updated lock data after role update] **************** -ok: [localhost] => {} - -MSG: - -Updated lock data after updating user_provision: - - current_role: user_provision - - role_completed: True - - completed_roles: {'base_os': {'completed': True, 'completed_at': '2025-07-24T15:29:50Z'}, 'lock_manager': {'completed': True, 'completed_at': '2025-07-24T15:27:44Z'}, 'secret_manager': {'completed': True, 'completed_at': '2025-07-24T15:30:03Z'}, 'user_provision': {'completed': True, 'completed_at': '2025-07-24T15:30:16Z'}, 'vdi_tool': {'completed': False}, 'vnc': {'completed': False}} - - -TASK [lock_manager : Get current users updated list] *************************** -ok: [localhost] => { - "ansible_facts": { - "current_users_updated": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Get new usernames list] *********************************** -ok: [localhost] => { - "ansible_facts": { - "new_usernames": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Combine users updated lists] ****************************** -ok: [localhost] => { - "ansible_facts": { - "combined_users_updated": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Debug user deduplication] ********************************* -ok: [localhost] => {} - -MSG: - -User deduplication for user_provision: - - current_users_updated: ['alice', 'bob'] - - new_usernames: ['alice', 'bob'] - - combined_users_updated: ['alice', 'bob'] - - -TASK [lock_manager : Get current user secrets for list update] ***************** -ok: [localhost] => { - "ansible_facts": { - "current_user_secrets_for_list": { - "last_secret_check": "2025-07-24T15:30:16Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Update users updated list] ******************************** -ok: [localhost] => { - "ansible_facts": { - "updated_user_secrets_with_list": { - "last_secret_check": "2025-07-24T15:30:16Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Apply users updated list] ********************************* -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:16Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:16Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Get current user management data] ************************* -ok: [localhost] => { - "ansible_facts": { - "current_user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - } - }, - "changed": false -} - -TASK [lock_manager : Get current usernames list] ******************************* -ok: [localhost] => { - "ansible_facts": { - "current_usernames": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Update user management data] ****************************** -ok: [localhost] => { - "ansible_facts": { - "updated_user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - } - }, - "changed": false -} - -TASK [lock_manager : Apply user management update] ***************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:16Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:16Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Debug user management] ************************************ -ok: [localhost] => {} - -MSG: - -User management for user_provision: - - current_users: ['alice', 'bob'] - - previous_users: ['alice', 'bob'] - - removed_users: [] - - -TASK [lock_manager : Check if all roles are completed] ************************* -ok: [localhost] => { - "ansible_facts": { - "all_roles_completed": false - }, - "changed": false -} - -TASK [lock_manager : Update setup status based on completion] ****************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:16Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:16Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Debug setup status update] ******************************** -ok: [localhost] => {} - -MSG: - -Setup status update: - - all_roles_completed: False - - setup_status: configuring - - completed_roles_count: 4 - - total_roles_count: 6 - - -TASK [lock_manager : Write updated lock file] ********************************** -changed: [localhost] => { - "changed": true, - "checksum": "ca35350cf060981f0667398fcb15c8e12423a97e", - "dest": "/opt/vdi-setup/.vdi-lock.yaml", - "gid": 0, - "group": "root", - "md5sum": "7ed7d0c10ced955765662b1feee9212a", - "mode": "0644", - "owner": "root", - "size": 1024, - "src": "/root/.ansible/tmp/ansible-tmp-1753371017.8474565-25936-21002111182327/source", - "state": "file", - "uid": 0 -} - -TASK [lock_manager : Debug VM metadata update] ********************************* -ok: [localhost] => {} - -MSG: - -VM metadata update info: - - hostname: vdi-test-scott-rocky-0 - - zone: us-central1-a - - lock_file_changed: True - - -TASK [lock_manager : Get lock file content as base64] ************************** -changed: [localhost] => { - "changed": true, - "cmd": "cat /opt/vdi-setup/.vdi-lock.yaml | base64 -w 0", - "delta": "0:00:00.003777", - "end": "2025-07-24 15:30:18.389398", - "rc": 0, - "start": "2025-07-24 15:30:18.385621" -} - -STDOUT: - -dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiB0cnVlCiAgICAgIGNvbXBsZXRlZF9hdDogJzIwMjUtMDctMjRUMTU6MzA6MTZaJwogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogZmFsc2UKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiBmYWxzZQogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNTozMDoxNlonCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogY29uZmlndXJpbmcKICB1c2VyX21hbmFnZW1lbnQ6CiAgICBjdXJyZW50X3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHByZXZpb3VzX3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHJlbW92ZWRfdXNlcnM6IFtdCiAgdXNlcl9zZWNyZXRzX3N0YXR1czoKICAgIGxhc3Rfc2VjcmV0X2NoZWNrOiAnMjAyNS0wNy0yNFQxNTozMDoxNlonCiAgICB1c2VyX3NlY3JldHNfaGFzaDogNmE2YTAyZDNkMDk5MzNjYTI4MmFlMmIwMjMzNjM0M2M5ZDczNzAxZTQ1YmQ4MTQ3NWZiZDkyZGNmOWQyZjNiNgogICAgdXNlcnNfdXBkYXRlZDoKICAgIC0gYWxpY2UKICAgIC0gYm9iCg== - -TASK [lock_manager : Add VDI lock file content to VM metadata] ***************** -changed: [localhost] => { - "changed": true, - "cmd": [ - "gcloud", - "compute", - "instances", - "add-metadata", - "vdi-test-scott-rocky-0", - "--metadata", - "vdi-lock-content=dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiB0cnVlCiAgICAgIGNvbXBsZXRlZF9hdDogJzIwMjUtMDctMjRUMTU6MzA6MTZaJwogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogZmFsc2UKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiBmYWxzZQogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNTozMDoxNlonCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogY29uZmlndXJpbmcKICB1c2VyX21hbmFnZW1lbnQ6CiAgICBjdXJyZW50X3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHByZXZpb3VzX3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHJlbW92ZWRfdXNlcnM6IFtdCiAgdXNlcl9zZWNyZXRzX3N0YXR1czoKICAgIGxhc3Rfc2VjcmV0X2NoZWNrOiAnMjAyNS0wNy0yNFQxNTozMDoxNlonCiAgICB1c2VyX3NlY3JldHNfaGFzaDogNmE2YTAyZDNkMDk5MzNjYTI4MmFlMmIwMjMzNjM0M2M5ZDczNzAxZTQ1YmQ4MTQ3NWZiZDkyZGNmOWQyZjNiNgogICAgdXNlcnNfdXBkYXRlZDoKICAgIC0gYWxpY2UKICAgIC0gYm9iCg==", - "--zone=us-central1-a" - ], - "delta": "0:00:04.229183", - "end": "2025-07-24 15:30:22.808719", - "failed_when_result": false, - "rc": 0, - "start": "2025-07-24 15:30:18.579536" -} - -STDERR: - -Updated [https://www.googleapis.com/compute/v1/projects/hpc-discovery-external/zones/us-central1-a/instances/vdi-test-scott-rocky-0]. - -TASK [lock_manager : Debug lock file write] ************************************ -ok: [localhost] => {} - -MSG: - -Lock file written: True - -TASK [lock_manager : Verify lock file integrity] ******************************* -ok: [localhost] => { - "changed": false, - "cmd": [ - "python3", - "-c", - "import yaml; yaml.safe_load(open('/opt/vdi-setup/.vdi-lock.yaml'))" - ], - "delta": "0:00:00.040331", - "end": "2025-07-24 15:30:23.092904", - "rc": 0, - "start": "2025-07-24 15:30:23.052573" -} - -TASK [lock_manager : Debug lock file validation] ******************************* -ok: [localhost] => {} - -MSG: - -Lock file validation: True - -TASK [lock_manager : Check if VDI monitor service exists] ********************** -ok: [localhost] => { - "changed": false, - "stat": { - "atime": 1753370874.1548011, - "attr_flags": "", - "attributes": [], - "block_size": 4096, - "blocks": 8, - "charset": "us-ascii", - "checksum": "f6de22707f8d9248014f679a2f1930451e2bf635", - "ctime": 1753370872.6256874, - "dev": 2050, - "device_type": 0, - "executable": false, - "exists": true, - "gid": 0, - "gr_name": "root", - "inode": 285214140, - "isblk": false, - "ischr": false, - "isdir": false, - "isfifo": false, - "isgid": false, - "islnk": false, - "isreg": true, - "issock": false, - "isuid": false, - "mimetype": "text/plain", - "mode": "0644", - "mtime": 1753370872.4766762, - "nlink": 1, - "path": "/etc/systemd/system/vdi-monitor.service", - "pw_name": "root", - "readable": true, - "rgrp": true, - "roth": true, - "rusr": true, - "size": 283, - "uid": 0, - "version": "1338188335", - "wgrp": false, - "woth": false, - "writeable": true, - "wusr": true, - "xgrp": false, - "xoth": false, - "xusr": false - } -} - -TASK [lock_manager : Debug VDI monitor setup conditions] *********************** -ok: [localhost] => {} - -MSG: - -VDI monitor setup conditions: - - current_role: user_provision - - vdi_monitor_service_exists: True - - lock_file_creation.changed: False - - will_setup_monitor: False - - -TASK [lock_manager : Setup VDI monitoring service (first time only)] *********** -skipping: [localhost] => { - "changed": false, - "false_condition": "not vdi_monitor_service_stat.stat.exists", - "skip_reason": "Conditional result was False" -} - -TASK [lock_manager : Fail if lock file is invalid] ***************************** -skipping: [localhost] => { - "changed": false, - "false_condition": "lock_file_validation.rc != 0", - "skip_reason": "Conditional result was False" -} - -TASK [vnc : Set current role for lock manager check] *************************** -ok: [localhost] => { - "ansible_facts": { - "current_role": "vnc" - }, - "changed": false -} - -TASK [lock_manager : Check if lock file exists] ******************************** -ok: [localhost] => { - "changed": false, - "stat": { - "atime": 1753371018.3885562, - "attr_flags": "", - "attributes": [], - "block_size": 4096, - "blocks": 8, - "charset": "us-ascii", - "checksum": "ca35350cf060981f0667398fcb15c8e12423a97e", - "ctime": 1753371018.1625395, - "dev": 2050, - "device_type": 0, - "executable": false, - "exists": true, - "gid": 0, - "gr_name": "root", - "inode": 117754419, - "isblk": false, - "ischr": false, - "isdir": false, - "isfifo": false, - "isgid": false, - "islnk": false, - "isreg": true, - "issock": false, - "isuid": false, - "mimetype": "text/plain", - "mode": "0644", - "mtime": 1753371018.0155284, - "nlink": 1, - "path": "/opt/vdi-setup/.vdi-lock.yaml", - "pw_name": "root", - "readable": true, - "rgrp": true, - "roth": true, - "rusr": true, - "size": 1024, - "uid": 0, - "version": "3086664638", - "wgrp": false, - "woth": false, - "writeable": true, - "wusr": true, - "xgrp": false, - "xoth": false, - "xusr": false - } -} - -TASK [lock_manager : Debug lock file existence] ******************************** -ok: [localhost] => {} - -MSG: - -Lock file exists: True - -TASK [lock_manager : Load existing lock file] ********************************** -ok: [localhost] => { - "ansible_facts": { - "vdi_setup_status": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:16Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:16Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "ansible_included_var_files": [ - "/opt/vdi-setup/.vdi-lock.yaml" - ], - "changed": false -} - -TASK [lock_manager : Debug lock file load result] ****************************** -ok: [localhost] => {} - -MSG: - -Lock file loaded successfully: True - -TASK [lock_manager : Calculate current deployment hash] ************************ -ok: [localhost] => { - "ansible_facts": { - "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab" - }, - "changed": false -} - -TASK [lock_manager : Debug deployment hash calculation] ************************ -ok: [localhost] => {} - -MSG: - -Current deployment hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab -Deployment components: - - deployment_name: vdi-test-scott - - project_id: hpc-discovery-external - - vdi_tool: guacamole - - vnc_flavor: tigervnc - - user_provision: local_users - - vdi_webapp_port: 8081 - - vdi_resolution: 1920x1080 - - vdi_users count: 2 - - -TASK [lock_manager : Calculate current user secrets hash] ********************** -ok: [localhost] => { - "ansible_facts": { - "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" - }, - "changed": false -} - -TASK [lock_manager : Debug user secrets hash calculation] ********************** -ok: [localhost] => {} - -MSG: - -Current user secrets hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 -User configuration components: - - vdi_users (blueprint): [{"username": "alice", "port": 5901}, {"username": "bob", "port": 5902, "secret_name": "a-password-for-bob"}] - - -TASK [lock_manager : Set default values for missing lock file] ***************** -skipping: [localhost] => { - "changed": false, - "false_condition": "not lock_file_stat.stat.exists", - "skip_reason": "Conditional result was False" -} - -TASK [lock_manager : Debug lock file status] *********************************** -ok: [localhost] => {} - -MSG: - -Lock file status: - - exists: True - - deployment_hash: none - - user_secrets_hash: none - - force_rerun: False - - -TASK [lock_manager : Check if lock file exists] ******************************** -ok: [localhost] => { - "ansible_facts": { - "lock_file_exists": true - }, - "changed": false -} - -TASK [lock_manager : Check if current role is completed] *********************** -ok: [localhost] => { - "ansible_facts": { - "current_role_completed": false - }, - "changed": false -} - -TASK [lock_manager : Check if deployment hash matches] ************************* -ok: [localhost] => { - "ansible_facts": { - "deployment_hash_matches": false - }, - "changed": false -} - -TASK [lock_manager : Check if force rerun is enabled] ************************** -ok: [localhost] => { - "ansible_facts": { - "force_rerun_enabled": false - }, - "changed": false -} - -TASK [lock_manager : Determine if role should run] ***************************** -ok: [localhost] => { - "ansible_facts": { - "role_should_run": true - }, - "changed": false -} - -TASK [lock_manager : Debug role should run calculation] ************************ -ok: [localhost] => {} - -MSG: - -Role should run calculation for vnc: - - not lock_file_exists: False - - not current_role_completed: True - - not deployment_hash_matches: True - - force_rerun_enabled: False - - role_should_run: True - - -TASK [lock_manager : Debug role execution decision] **************************** -ok: [localhost] => {} - -MSG: - -Role execution decision for vnc: - - current_role: vnc - - ansible_role_name: lock_manager - - role_should_run: True - - lock_file_exists: True - - current_role_completed: False - - deployment_hash_matches: False - - force_rerun_enabled: False - - -TASK [lock_manager : Check if user configuration has changed] ****************** -ok: [localhost] => { - "ansible_facts": { - "user_config_changed": true - }, - "changed": false -} - -TASK [lock_manager : Set user secrets changed flag (will be updated by secret_manager role)] *** -ok: [localhost] => { - "ansible_facts": { - "user_secrets_changed": true - }, - "changed": false -} - -TASK [lock_manager : Debug user secrets change detection] ********************** -ok: [localhost] => {} - -MSG: - -User configuration change detection: - - user_config_changed: True - - user_secrets_changed: True - - stored_hash: none - - current_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 - - -TASK [lock_manager : Set lock check result] ************************************ -ok: [localhost] => { - "ansible_facts": { - "lock_check_result": { - "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "existing_deployment_hash": "none", - "existing_user_secrets_hash": "none", - "force_rerun": false, - "role_should_run": true, - "user_secrets_changed": true - } - }, - "changed": false -} - -TASK [lock_manager : Debug final lock check result] **************************** -ok: [localhost] => {} - -MSG: - -Final lock check result: - - role_should_run: True - - deployment_hash_changed: True - - user_secrets_changed: True - - force_rerun: False - - -TASK [vnc : Set role_should_run from lock check result] ************************ -ok: [localhost] => { - "ansible_facts": { - "role_should_run": true - }, - "changed": false -} - -TASK [vnc : Skip vnc tasks if role should not run] ***************************** -skipping: [localhost] => { - "false_condition": "not role_should_run" -} - -TASK [vnc : Install TigerVNC] ************************************************** -included: /tmp/vdi/roles/vnc/tasks/tigervnc.yaml for localhost - -TASK [vnc : Configure per-user VNC settings] *********************************** -included: /tmp/vdi/roles/vnc/tasks/vnc_user_config.yaml for localhost => (item={'username': 'alice', 'port': 5901, 'password': 'xc61q4jU3mUCo84M', 'vncserver_password': '4JB1YHUQ', 'display_number': 1}) -included: /tmp/vdi/roles/vnc/tasks/vnc_user_config.yaml for localhost => (item={'username': 'bob', 'port': 5902, 'secret_name': 'a-password-for-bob', 'password': 'B0bp4ssw0rd123', 'vncserver_password': '7DmxvFp2', 'display_number': 2}) - -TASK [vnc : Ensure VNC configuration directory exists] ************************* -changed: [localhost] => { - "changed": true, - "gid": 1001, - "group": "vdiusers", - "mode": "0700", - "owner": "alice", - "path": "/home/alice/.vnc", - "size": 6, - "state": "directory", - "uid": 1000 -} - -TASK [vnc : Write VNC xstartup for alice (Ubuntu/Debian)] ********************** -skipping: [localhost] => { - "changed": false, - "false_condition": "ansible_distribution in [\"Ubuntu\", \"Debian\"]", - "skip_reason": "Conditional result was False" -} - -TASK [vnc : Write VNC password file for alice] ********************************* -changed: [localhost] => { - "changed": true, - "cmd": "echo \"4JB1YHUQ\" | vncpasswd -f > \"/home/alice/.vnc/passwd\"\n", - "delta": "0:00:00.009600", - "end": "2025-07-24 15:30:24.957369", - "rc": 0, - "start": "2025-07-24 15:30:24.947769" -} - -TASK [vnc : Fix passwd file permissions] *************************************** -changed: [localhost] => { - "changed": true, - "gid": 1001, - "group": "vdiusers", - "mode": "0600", - "owner": "alice", - "path": "/home/alice/.vnc/passwd", - "size": 8, - "state": "file", - "uid": 1000 -} - -TASK [vnc : Copy per-user VNC config] ****************************************** -changed: [localhost] => { - "changed": true, - "checksum": "e5a3133663f35d3a79dc06917090845ab424c412", - "dest": "/home/alice/.vnc/config", - "gid": 1001, - "group": "vdiusers", - "md5sum": "15686602524e414a443391251ac46b4a", - "mode": "0644", - "owner": "alice", - "size": 32, - "src": "/root/.ansible/tmp/ansible-tmp-1753371025.2160182-26228-110817536086157/source", - "state": "file", - "uid": 1000 -} - -TASK [vnc : Ensure VNC configuration directory exists] ************************* -changed: [localhost] => { - "changed": true, - "gid": 1001, - "group": "vdiusers", - "mode": "0700", - "owner": "bob", - "path": "/home/bob/.vnc", - "size": 6, - "state": "directory", - "uid": 1001 -} - -TASK [vnc : Write VNC xstartup for bob (Ubuntu/Debian)] ************************ -skipping: [localhost] => { - "changed": false, - "false_condition": "ansible_distribution in [\"Ubuntu\", \"Debian\"]", - "skip_reason": "Conditional result was False" -} - -TASK [vnc : Write VNC password file for bob] *********************************** -changed: [localhost] => { - "changed": true, - "cmd": "echo \"7DmxvFp2\" | vncpasswd -f > \"/home/bob/.vnc/passwd\"\n", - "delta": "0:00:00.007207", - "end": "2025-07-24 15:30:26.051034", - "rc": 0, - "start": "2025-07-24 15:30:26.043827" -} - -TASK [vnc : Fix passwd file permissions] *************************************** -changed: [localhost] => { - "changed": true, - "gid": 1001, - "group": "vdiusers", - "mode": "0600", - "owner": "bob", - "path": "/home/bob/.vnc/passwd", - "size": 8, - "state": "file", - "uid": 1001 -} - -TASK [vnc : Copy per-user VNC config] ****************************************** -changed: [localhost] => { - "changed": true, - "checksum": "e5a3133663f35d3a79dc06917090845ab424c412", - "dest": "/home/bob/.vnc/config", - "gid": 1001, - "group": "vdiusers", - "md5sum": "15686602524e414a443391251ac46b4a", - "mode": "0644", - "owner": "bob", - "size": 32, - "src": "/root/.ansible/tmp/ansible-tmp-1753371026.302065-26344-216915276880984/source", - "state": "file", - "uid": 1001 -} - -TASK [vnc : Template TigerVNC users file] ************************************** -changed: [localhost] => { - "changed": true, - "checksum": "07719f46faaccbdf742785d5d2310a55a7352cdb", - "dest": "/etc/tigervnc/vncserver.users", - "gid": 0, - "group": "root", - "md5sum": "4e1b63b33d08dda6c1a7dde083fe9854", - "mode": "0644", - "owner": "root", - "size": 16, - "src": "/root/.ansible/tmp/ansible-tmp-1753371026.663593-26366-244603176491990/source", - "state": "file", - "uid": 0 -} - -TASK [vnc : Deploy per-user TigerVNC systemd unit (Ubuntu/Debian)] ************* -skipping: [localhost] => (item={'username': 'alice', 'port': 5901, 'password': 'xc61q4jU3mUCo84M', 'vncserver_password': '4JB1YHUQ', 'display_number': 1}) => { - "ansible_loop_var": "item", - "changed": false, - "false_condition": "ansible_distribution in [\"Ubuntu\", \"Debian\"]", - "item": { - "display_number": 1, - "password": "xc61q4jU3mUCo84M", - "port": 5901, - "username": "alice", - "vncserver_password": "4JB1YHUQ" - }, - "skip_reason": "Conditional result was False" -} -skipping: [localhost] => (item={'username': 'bob', 'port': 5902, 'secret_name': 'a-password-for-bob', 'password': 'B0bp4ssw0rd123', 'vncserver_password': '7DmxvFp2', 'display_number': 2}) => { - "ansible_loop_var": "item", - "changed": false, - "false_condition": "ansible_distribution in [\"Ubuntu\", \"Debian\"]", - "item": { - "display_number": 2, - "password": "B0bp4ssw0rd123", - "port": 5902, - "secret_name": "a-password-for-bob", - "username": "bob", - "vncserver_password": "7DmxvFp2" - }, - "skip_reason": "Conditional result was False" -} -skipping: [localhost] => { - "changed": false -} - -MSG: - -All items skipped - -TASK [vnc : Deploy TigerVNC systemd unit (Rocky)] ****************************** -changed: [localhost] => { - "changed": true, - "checksum": "bd078a72af7384ad8cc0fdd595ea34d71f75969c", - "dest": "/etc/systemd/system/vncserver@.service", - "gid": 0, - "group": "root", - "md5sum": "a6a8e5fa35e07463b97f8750bab854fa", - "mode": "0644", - "owner": "root", - "size": 268, - "src": "/root/.ansible/tmp/ansible-tmp-1753371027.0719545-26392-180913534199741/source", - "state": "file", - "uid": 0 -} - -TASK [vnc : Reload systemd daemon] ********************************************* -ok: [localhost] => { - "changed": false, - "name": null, - "status": {} -} - -TASK [vnc : Enable & start TigerVNC (Ubuntu/Debian)] *************************** -skipping: [localhost] => (item={'username': 'alice', 'port': 5901, 'password': 'xc61q4jU3mUCo84M', 'vncserver_password': '4JB1YHUQ', 'display_number': 1}) => { - "ansible_loop_var": "item", - "changed": false, - "false_condition": "ansible_distribution in [\"Ubuntu\", \"Debian\"]", - "item": { - "display_number": 1, - "password": "xc61q4jU3mUCo84M", - "port": 5901, - "username": "alice", - "vncserver_password": "4JB1YHUQ" - }, - "skip_reason": "Conditional result was False" -} -skipping: [localhost] => (item={'username': 'bob', 'port': 5902, 'secret_name': 'a-password-for-bob', 'password': 'B0bp4ssw0rd123', 'vncserver_password': '7DmxvFp2', 'display_number': 2}) => { - "ansible_loop_var": "item", - "changed": false, - "false_condition": "ansible_distribution in [\"Ubuntu\", \"Debian\"]", - "item": { - "display_number": 2, - "password": "B0bp4ssw0rd123", - "port": 5902, - "secret_name": "a-password-for-bob", - "username": "bob", - "vncserver_password": "7DmxvFp2" - }, - "skip_reason": "Conditional result was False" -} -skipping: [localhost] => { - "changed": false -} - -MSG: - -All items skipped - -TASK [vnc : Enable & start TigerVNC (Rocky)] *********************************** -changed: [localhost] => (item={'username': 'alice', 'port': 5901, 'password': 'xc61q4jU3mUCo84M', 'vncserver_password': '4JB1YHUQ', 'display_number': 1}) => { - "ansible_loop_var": "item", - "changed": true, - "enabled": true, - "item": { - "display_number": 1, - "password": "xc61q4jU3mUCo84M", - "port": 5901, - "username": "alice", - "vncserver_password": "4JB1YHUQ" - }, - "name": "vncserver@:1", - "state": "started", - "status": { - "ActiveEnterTimestampMonotonic": "0", - "ActiveExitTimestampMonotonic": "0", - "ActiveState": "inactive", - "After": "syslog.target basic.target sysinit.target system-vncserver.slice network.target systemd-journald.socket", - "AllowIsolate": "no", - "AllowedCPUs": "", - "AllowedMemoryNodes": "", - "AmbientCapabilities": "", - "AssertResult": "no", - "AssertTimestampMonotonic": "0", - "Before": "shutdown.target", - "BlockIOAccounting": "no", - "BlockIOWeight": "[not set]", - "CPUAccounting": "no", - "CPUAffinity": "", - "CPUAffinityFromNUMA": "no", - "CPUQuotaPerSecUSec": "infinity", - "CPUQuotaPeriodUSec": "infinity", - "CPUSchedulingPolicy": "0", - "CPUSchedulingPriority": "0", - "CPUSchedulingResetOnFork": "no", - "CPUShares": "[not set]", - "CPUUsageNSec": "[not set]", - "CPUWeight": "[not set]", - "CacheDirectoryMode": "0755", - "CanFreeze": "yes", - "CanIsolate": "no", - "CanReload": "no", - "CanStart": "yes", - "CanStop": "yes", - "CapabilityBoundingSet": "cap_chown cap_dac_override cap_dac_read_search cap_fowner cap_fsetid cap_kill cap_setgid cap_setuid cap_setpcap cap_linux_immutable cap_net_bind_service cap_net_broadcast cap_net_admin cap_net_raw cap_ipc_lock cap_ipc_owner cap_sys_module cap_sys_rawio cap_sys_chroot cap_sys_ptrace cap_sys_pacct cap_sys_admin cap_sys_boot cap_sys_nice cap_sys_resource cap_sys_time cap_sys_tty_config cap_mknod cap_lease cap_audit_write cap_audit_control cap_setfcap cap_mac_override cap_mac_admin cap_syslog cap_wake_alarm cap_block_suspend cap_audit_read cap_perfmon cap_bpf", - "CollectMode": "inactive", - "ConditionResult": "no", - "ConditionTimestampMonotonic": "0", - "ConfigurationDirectoryMode": "0755", - "Conflicts": "shutdown.target", - "ControlPID": "0", - "DefaultDependencies": "yes", - "DefaultMemoryLow": "0", - "DefaultMemoryMin": "0", - "Delegate": "no", - "Description": "TigerVNC server for user :1", - "DevicePolicy": "auto", - "DynamicUser": "no", - "EffectiveCPUs": "", - "EffectiveMemoryNodes": "", - "ExecMainCode": "0", - "ExecMainExitTimestampMonotonic": "0", - "ExecMainPID": "0", - "ExecMainStartTimestampMonotonic": "0", - "ExecMainStatus": "0", - "ExecStart": "{ path=/usr/libexec/vncsession-start ; argv[]=/usr/libexec/vncsession-start :1 ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", - "ExecStop": "{ path=/bin/true ; argv[]=/bin/true ; ignore_errors=yes ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", - "FailureAction": "none", - "FileDescriptorStoreMax": "0", - "FragmentPath": "/etc/systemd/system/vncserver@.service", - "FreezerState": "running", - "GID": "[not set]", - "GuessMainPID": "yes", - "IOAccounting": "no", - "IOSchedulingClass": "0", - "IOSchedulingPriority": "0", - "IOWeight": "[not set]", - "IPAccounting": "no", - "IPEgressBytes": "18446744073709551615", - "IPEgressPackets": "18446744073709551615", - "IPIngressBytes": "18446744073709551615", - "IPIngressPackets": "18446744073709551615", - "Id": "vncserver@:1.service", - "IgnoreOnIsolate": "no", - "IgnoreSIGPIPE": "yes", - "InactiveEnterTimestampMonotonic": "0", - "InactiveExitTimestampMonotonic": "0", - "JobRunningTimeoutUSec": "infinity", - "JobTimeoutAction": "none", - "JobTimeoutUSec": "infinity", - "KeyringMode": "private", - "KillMode": "control-group", - "KillSignal": "15", - "LimitAS": "infinity", - "LimitASSoft": "infinity", - "LimitCORE": "infinity", - "LimitCORESoft": "0", - "LimitCPU": "infinity", - "LimitCPUSoft": "infinity", - "LimitDATA": "infinity", - "LimitDATASoft": "infinity", - "LimitFSIZE": "infinity", - "LimitFSIZESoft": "infinity", - "LimitLOCKS": "infinity", - "LimitLOCKSSoft": "infinity", - "LimitMEMLOCK": "65536", - "LimitMEMLOCKSoft": "65536", - "LimitMSGQUEUE": "819200", - "LimitMSGQUEUESoft": "819200", - "LimitNICE": "0", - "LimitNICESoft": "0", - "LimitNOFILE": "262144", - "LimitNOFILESoft": "1024", - "LimitNPROC": "127383", - "LimitNPROCSoft": "127383", - "LimitRSS": "infinity", - "LimitRSSSoft": "infinity", - "LimitRTPRIO": "0", - "LimitRTPRIOSoft": "0", - "LimitRTTIME": "infinity", - "LimitRTTIMESoft": "infinity", - "LimitSIGPENDING": "127383", - "LimitSIGPENDINGSoft": "127383", - "LimitSTACK": "infinity", - "LimitSTACKSoft": "8388608", - "LoadState": "loaded", - "LockPersonality": "no", - "LogLevelMax": "-1", - "LogRateLimitBurst": "0", - "LogRateLimitIntervalUSec": "0", - "LogsDirectoryMode": "0755", - "MainPID": "0", - "MemoryAccounting": "yes", - "MemoryCurrent": "[not set]", - "MemoryDenyWriteExecute": "no", - "MemoryHigh": "infinity", - "MemoryLimit": "infinity", - "MemoryLow": "0", - "MemoryMax": "infinity", - "MemoryMin": "0", - "MemorySwapMax": "infinity", - "MountAPIVFS": "no", - "MountFlags": "", - "NFileDescriptorStore": "0", - "NRestarts": "0", - "NUMAMask": "", - "NUMAPolicy": "n/a", - "Names": "vncserver@:1.service", - "NeedDaemonReload": "no", - "Nice": "0", - "NoNewPrivileges": "no", - "NonBlocking": "no", - "NotifyAccess": "none", - "OOMScoreAdjust": "0", - "OnFailureJobMode": "replace", - "PIDFile": "/run/vncsession-:1.pid", - "PermissionsStartOnly": "no", - "Perpetual": "no", - "PrivateDevices": "no", - "PrivateMounts": "no", - "PrivateNetwork": "no", - "PrivateTmp": "no", - "PrivateUsers": "no", - "ProtectControlGroups": "no", - "ProtectHome": "no", - "ProtectKernelModules": "no", - "ProtectKernelTunables": "no", - "ProtectSystem": "no", - "RefuseManualStart": "no", - "RefuseManualStop": "no", - "RemainAfterExit": "no", - "RemoveIPC": "no", - "Requires": "sysinit.target system-vncserver.slice", - "Restart": "always", - "RestartUSec": "15s", - "RestrictNamespaces": "no", - "RestrictRealtime": "no", - "RestrictSUIDSGID": "no", - "Result": "success", - "RootDirectoryStartOnly": "no", - "RuntimeDirectoryMode": "0755", - "RuntimeDirectoryPreserve": "no", - "RuntimeMaxUSec": "infinity", - "SameProcessGroup": "no", - "SecureBits": "0", - "SendSIGHUP": "no", - "SendSIGKILL": "yes", - "Slice": "system-vncserver.slice", - "StandardError": "inherit", - "StandardInput": "null", - "StandardInputData": "", - "StandardOutput": "journal", - "StartLimitAction": "none", - "StartLimitBurst": "5", - "StartLimitIntervalUSec": "10s", - "StartupBlockIOWeight": "[not set]", - "StartupCPUShares": "[not set]", - "StartupCPUWeight": "[not set]", - "StartupIOWeight": "[not set]", - "StateChangeTimestampMonotonic": "0", - "StateDirectoryMode": "0755", - "StatusErrno": "0", - "StopWhenUnneeded": "no", - "SubState": "dead", - "SuccessAction": "none", - "SyslogFacility": "3", - "SyslogLevel": "6", - "SyslogLevelPrefix": "yes", - "SyslogPriority": "30", - "SystemCallErrorNumber": "0", - "TTYReset": "no", - "TTYVHangup": "no", - "TTYVTDisallocate": "no", - "TasksAccounting": "yes", - "TasksCurrent": "[not set]", - "TasksMax": "203813", - "TimeoutStartUSec": "1min 30s", - "TimeoutStopUSec": "1min 30s", - "TimerSlackNSec": "50000", - "Transient": "no", - "Type": "forking", - "UID": "[not set]", - "UMask": "0022", - "UnitFilePreset": "disabled", - "UnitFileState": "disabled", - "UtmpMode": "init", - "WatchdogTimestampMonotonic": "0", - "WatchdogUSec": "0" - } -} -changed: [localhost] => (item={'username': 'bob', 'port': 5902, 'secret_name': 'a-password-for-bob', 'password': 'B0bp4ssw0rd123', 'vncserver_password': '7DmxvFp2', 'display_number': 2}) => { - "ansible_loop_var": "item", - "changed": true, - "enabled": true, - "item": { - "display_number": 2, - "password": "B0bp4ssw0rd123", - "port": 5902, - "secret_name": "a-password-for-bob", - "username": "bob", - "vncserver_password": "7DmxvFp2" - }, - "name": "vncserver@:2", - "state": "started", - "status": { - "ActiveEnterTimestampMonotonic": "0", - "ActiveExitTimestampMonotonic": "0", - "ActiveState": "inactive", - "After": "syslog.target sysinit.target basic.target system-vncserver.slice systemd-journald.socket network.target", - "AllowIsolate": "no", - "AllowedCPUs": "", - "AllowedMemoryNodes": "", - "AmbientCapabilities": "", - "AssertResult": "no", - "AssertTimestampMonotonic": "0", - "Before": "shutdown.target", - "BlockIOAccounting": "no", - "BlockIOWeight": "[not set]", - "CPUAccounting": "no", - "CPUAffinity": "", - "CPUAffinityFromNUMA": "no", - "CPUQuotaPerSecUSec": "infinity", - "CPUQuotaPeriodUSec": "infinity", - "CPUSchedulingPolicy": "0", - "CPUSchedulingPriority": "0", - "CPUSchedulingResetOnFork": "no", - "CPUShares": "[not set]", - "CPUUsageNSec": "[not set]", - "CPUWeight": "[not set]", - "CacheDirectoryMode": "0755", - "CanFreeze": "yes", - "CanIsolate": "no", - "CanReload": "no", - "CanStart": "yes", - "CanStop": "yes", - "CapabilityBoundingSet": "cap_chown cap_dac_override cap_dac_read_search cap_fowner cap_fsetid cap_kill cap_setgid cap_setuid cap_setpcap cap_linux_immutable cap_net_bind_service cap_net_broadcast cap_net_admin cap_net_raw cap_ipc_lock cap_ipc_owner cap_sys_module cap_sys_rawio cap_sys_chroot cap_sys_ptrace cap_sys_pacct cap_sys_admin cap_sys_boot cap_sys_nice cap_sys_resource cap_sys_time cap_sys_tty_config cap_mknod cap_lease cap_audit_write cap_audit_control cap_setfcap cap_mac_override cap_mac_admin cap_syslog cap_wake_alarm cap_block_suspend cap_audit_read cap_perfmon cap_bpf", - "CollectMode": "inactive", - "ConditionResult": "no", - "ConditionTimestampMonotonic": "0", - "ConfigurationDirectoryMode": "0755", - "Conflicts": "shutdown.target", - "ControlPID": "0", - "DefaultDependencies": "yes", - "DefaultMemoryLow": "0", - "DefaultMemoryMin": "0", - "Delegate": "no", - "Description": "TigerVNC server for user :2", - "DevicePolicy": "auto", - "DynamicUser": "no", - "EffectiveCPUs": "", - "EffectiveMemoryNodes": "", - "ExecMainCode": "0", - "ExecMainExitTimestampMonotonic": "0", - "ExecMainPID": "0", - "ExecMainStartTimestampMonotonic": "0", - "ExecMainStatus": "0", - "ExecStart": "{ path=/usr/libexec/vncsession-start ; argv[]=/usr/libexec/vncsession-start :2 ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", - "ExecStop": "{ path=/bin/true ; argv[]=/bin/true ; ignore_errors=yes ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", - "FailureAction": "none", - "FileDescriptorStoreMax": "0", - "FragmentPath": "/etc/systemd/system/vncserver@.service", - "FreezerState": "running", - "GID": "[not set]", - "GuessMainPID": "yes", - "IOAccounting": "no", - "IOSchedulingClass": "0", - "IOSchedulingPriority": "0", - "IOWeight": "[not set]", - "IPAccounting": "no", - "IPEgressBytes": "18446744073709551615", - "IPEgressPackets": "18446744073709551615", - "IPIngressBytes": "18446744073709551615", - "IPIngressPackets": "18446744073709551615", - "Id": "vncserver@:2.service", - "IgnoreOnIsolate": "no", - "IgnoreSIGPIPE": "yes", - "InactiveEnterTimestampMonotonic": "0", - "InactiveExitTimestampMonotonic": "0", - "JobRunningTimeoutUSec": "infinity", - "JobTimeoutAction": "none", - "JobTimeoutUSec": "infinity", - "KeyringMode": "private", - "KillMode": "control-group", - "KillSignal": "15", - "LimitAS": "infinity", - "LimitASSoft": "infinity", - "LimitCORE": "infinity", - "LimitCORESoft": "0", - "LimitCPU": "infinity", - "LimitCPUSoft": "infinity", - "LimitDATA": "infinity", - "LimitDATASoft": "infinity", - "LimitFSIZE": "infinity", - "LimitFSIZESoft": "infinity", - "LimitLOCKS": "infinity", - "LimitLOCKSSoft": "infinity", - "LimitMEMLOCK": "65536", - "LimitMEMLOCKSoft": "65536", - "LimitMSGQUEUE": "819200", - "LimitMSGQUEUESoft": "819200", - "LimitNICE": "0", - "LimitNICESoft": "0", - "LimitNOFILE": "262144", - "LimitNOFILESoft": "1024", - "LimitNPROC": "127383", - "LimitNPROCSoft": "127383", - "LimitRSS": "infinity", - "LimitRSSSoft": "infinity", - "LimitRTPRIO": "0", - "LimitRTPRIOSoft": "0", - "LimitRTTIME": "infinity", - "LimitRTTIMESoft": "infinity", - "LimitSIGPENDING": "127383", - "LimitSIGPENDINGSoft": "127383", - "LimitSTACK": "infinity", - "LimitSTACKSoft": "8388608", - "LoadState": "loaded", - "LockPersonality": "no", - "LogLevelMax": "-1", - "LogRateLimitBurst": "0", - "LogRateLimitIntervalUSec": "0", - "LogsDirectoryMode": "0755", - "MainPID": "0", - "MemoryAccounting": "yes", - "MemoryCurrent": "[not set]", - "MemoryDenyWriteExecute": "no", - "MemoryHigh": "infinity", - "MemoryLimit": "infinity", - "MemoryLow": "0", - "MemoryMax": "infinity", - "MemoryMin": "0", - "MemorySwapMax": "infinity", - "MountAPIVFS": "no", - "MountFlags": "", - "NFileDescriptorStore": "0", - "NRestarts": "0", - "NUMAMask": "", - "NUMAPolicy": "n/a", - "Names": "vncserver@:2.service", - "NeedDaemonReload": "no", - "Nice": "0", - "NoNewPrivileges": "no", - "NonBlocking": "no", - "NotifyAccess": "none", - "OOMScoreAdjust": "0", - "OnFailureJobMode": "replace", - "PIDFile": "/run/vncsession-:2.pid", - "PermissionsStartOnly": "no", - "Perpetual": "no", - "PrivateDevices": "no", - "PrivateMounts": "no", - "PrivateNetwork": "no", - "PrivateTmp": "no", - "PrivateUsers": "no", - "ProtectControlGroups": "no", - "ProtectHome": "no", - "ProtectKernelModules": "no", - "ProtectKernelTunables": "no", - "ProtectSystem": "no", - "RefuseManualStart": "no", - "RefuseManualStop": "no", - "RemainAfterExit": "no", - "RemoveIPC": "no", - "Requires": "system-vncserver.slice sysinit.target", - "Restart": "always", - "RestartUSec": "15s", - "RestrictNamespaces": "no", - "RestrictRealtime": "no", - "RestrictSUIDSGID": "no", - "Result": "success", - "RootDirectoryStartOnly": "no", - "RuntimeDirectoryMode": "0755", - "RuntimeDirectoryPreserve": "no", - "RuntimeMaxUSec": "infinity", - "SameProcessGroup": "no", - "SecureBits": "0", - "SendSIGHUP": "no", - "SendSIGKILL": "yes", - "Slice": "system-vncserver.slice", - "StandardError": "inherit", - "StandardInput": "null", - "StandardInputData": "", - "StandardOutput": "journal", - "StartLimitAction": "none", - "StartLimitBurst": "5", - "StartLimitIntervalUSec": "10s", - "StartupBlockIOWeight": "[not set]", - "StartupCPUShares": "[not set]", - "StartupCPUWeight": "[not set]", - "StartupIOWeight": "[not set]", - "StateChangeTimestampMonotonic": "0", - "StateDirectoryMode": "0755", - "StatusErrno": "0", - "StopWhenUnneeded": "no", - "SubState": "dead", - "SuccessAction": "none", - "SyslogFacility": "3", - "SyslogLevel": "6", - "SyslogLevelPrefix": "yes", - "SyslogPriority": "30", - "SystemCallErrorNumber": "0", - "TTYReset": "no", - "TTYVHangup": "no", - "TTYVTDisallocate": "no", - "TasksAccounting": "yes", - "TasksCurrent": "[not set]", - "TasksMax": "203813", - "TimeoutStartUSec": "1min 30s", - "TimeoutStopUSec": "1min 30s", - "TimerSlackNSec": "50000", - "Transient": "no", - "Type": "forking", - "UID": "[not set]", - "UMask": "0022", - "UnitFilePreset": "disabled", - "UnitFileState": "disabled", - "UtmpMode": "init", - "WatchdogTimestampMonotonic": "0", - "WatchdogUSec": "0" - } -} - -TASK [vnc : Install TightVNC] ************************************************** -skipping: [localhost] => { - "changed": false, - "false_condition": "vnc_flavor | lower == 'tightvnc'", - "skip_reason": "Conditional result was False" -} - -TASK [vnc : Set current role for lock manager completion] ********************** -ok: [localhost] => { - "ansible_facts": { - "current_role": "vnc", - "role_completed": true - }, - "changed": false -} - -TASK [lock_manager : Debug create lock operation] ****************************** -ok: [localhost] => {} - -MSG: - -Creating/updating lock file for role: vnc - -TASK [lock_manager : Check if lock file exists] ******************************** -ok: [localhost] => { - "changed": false, - "stat": { - "atime": 1753371018.3885562, - "attr_flags": "", - "attributes": [], - "block_size": 4096, - "blocks": 8, - "charset": "us-ascii", - "checksum": "ca35350cf060981f0667398fcb15c8e12423a97e", - "ctime": 1753371018.1625395, - "dev": 2050, - "device_type": 0, - "executable": false, - "exists": true, - "gid": 0, - "gr_name": "root", - "inode": 117754419, - "isblk": false, - "ischr": false, - "isdir": false, - "isfifo": false, - "isgid": false, - "islnk": false, - "isreg": true, - "issock": false, - "isuid": false, - "mimetype": "text/plain", - "mode": "0644", - "mtime": 1753371018.0155284, - "nlink": 1, - "path": "/opt/vdi-setup/.vdi-lock.yaml", - "pw_name": "root", - "readable": true, - "rgrp": true, - "roth": true, - "rusr": true, - "size": 1024, - "uid": 0, - "version": "3086664638", - "wgrp": false, - "woth": false, - "writeable": true, - "wusr": true, - "xgrp": false, - "xoth": false, - "xusr": false - } -} - -TASK [lock_manager : Debug lock file existence] ******************************** -ok: [localhost] => {} - -MSG: - -Lock file exists: True - -TASK [lock_manager : Create initial lock file structure] *********************** -skipping: [localhost] => { - "changed": false, - "false_condition": "not lock_file_stat.stat.exists", - "skip_reason": "Conditional result was False" -} - -TASK [lock_manager : Debug lock file creation] ********************************* -skipping: [localhost] => { - "false_condition": "debug | default(false) and not lock_file_stat.stat.exists" -} - -TASK [lock_manager : Load existing lock file data] ***************************** -ok: [localhost] => { - "ansible_facts": { - "vdi_setup_status": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:16Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:16Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "ansible_included_var_files": [ - "/opt/vdi-setup/.vdi-lock.yaml" - ], - "changed": false -} - -TASK [lock_manager : Get current lock data] ************************************ -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:16Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:16Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Update deployment hash] *********************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:16Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:16Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Get current timestamp] ************************************ -changed: [localhost] => { - "changed": true, - "cmd": [ - "date", - "-u", - "+%Y-%m-%dT%H:%M:%SZ" - ], - "delta": "0:00:00.020571", - "end": "2025-07-24 15:30:30.669137", - "rc": 0, - "start": "2025-07-24 15:30:30.648566" -} - -STDOUT: - -2025-07-24T15:30:30Z - -TASK [lock_manager : Update last updated timestamp] **************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:30Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:16Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Get current user secrets status] ************************** -ok: [localhost] => { - "ansible_facts": { - "current_user_secrets": { - "last_secret_check": "2025-07-24T15:30:16Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Create user secrets update] ******************************* -ok: [localhost] => { - "ansible_facts": { - "user_secrets_update": { - "last_secret_check": "2025-07-24T15:30:30Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" - } - }, - "changed": false -} - -TASK [lock_manager : Update user secrets hash] ********************************* -ok: [localhost] => { - "ansible_facts": { - "updated_user_secrets": { - "last_secret_check": "2025-07-24T15:30:30Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Apply user secrets update] ******************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:30Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:30Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Create role status entry] ********************************* -ok: [localhost] => { - "ansible_facts": { - "role_status_entry": { - "completed": true, - "completed_at": "2025-07-24T15:30:30Z" - } - }, - "changed": false -} - -TASK [lock_manager : Debug role status entry] ********************************** -ok: [localhost] => {} - -MSG: - -Role status entry for vnc: - - completed: True - - completed_at: 2025-07-24T15:30:30Z - - -TASK [lock_manager : Debug current lock data before role update] *************** -ok: [localhost] => {} - -MSG: - -Current lock data before updating vnc: - - current_lock_data: completed_roles: - base_os: - completed: true - completed_at: '2025-07-24T15:29:50Z' - lock_manager: - completed: true - completed_at: '2025-07-24T15:27:44Z' - secret_manager: - completed: true - completed_at: '2025-07-24T15:30:03Z' - user_provision: - completed: true - completed_at: '2025-07-24T15:30:16Z' - vdi_tool: - completed: false - vnc: - completed: false -created_at: '2025-07-24T15:27:41Z' -deployment_hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab -deployment_name: vdi-test-scott -force_rerun: false -last_updated: '2025-07-24T15:30:30Z' -lock_version: '1.0' -setup_status: configuring -user_management: - current_users: - - alice - - bob - previous_users: - - alice - - bob - removed_users: [] -user_secrets_status: - last_secret_check: '2025-07-24T15:30:30Z' - user_secrets_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 - users_updated: - - alice - - bob - - - current_completed_roles: base_os: - completed: true - completed_at: '2025-07-24T15:29:50Z' -lock_manager: - completed: true - completed_at: '2025-07-24T15:27:44Z' -secret_manager: - completed: true - completed_at: '2025-07-24T15:30:03Z' -user_provision: - completed: true - completed_at: '2025-07-24T15:30:16Z' -vdi_tool: - completed: false -vnc: - completed: false - - - role_completed: True - - current_role: vnc - - -TASK [lock_manager : Get current completed roles] ****************************** -ok: [localhost] => { - "ansible_facts": { - "current_completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": false - } - } - }, - "changed": false -} - -TASK [lock_manager : Update completed roles] *********************************** -ok: [localhost] => { - "ansible_facts": { - "updated_completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": true, - "completed_at": "2025-07-24T15:30:30Z" - } - } - }, - "changed": false -} - -TASK [lock_manager : Update lock data with completed roles] ******************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": true, - "completed_at": "2025-07-24T15:30:30Z" - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:30Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:30Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Debug updated lock data after role update] **************** -ok: [localhost] => {} - -MSG: - -Updated lock data after updating vnc: - - current_role: vnc - - role_completed: True - - completed_roles: {'base_os': {'completed': True, 'completed_at': '2025-07-24T15:29:50Z'}, 'lock_manager': {'completed': True, 'completed_at': '2025-07-24T15:27:44Z'}, 'secret_manager': {'completed': True, 'completed_at': '2025-07-24T15:30:03Z'}, 'user_provision': {'completed': True, 'completed_at': '2025-07-24T15:30:16Z'}, 'vdi_tool': {'completed': False}, 'vnc': {'completed': True, 'completed_at': '2025-07-24T15:30:30Z'}} - - -TASK [lock_manager : Get current users updated list] *************************** -ok: [localhost] => { - "ansible_facts": { - "current_users_updated": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Get new usernames list] *********************************** -ok: [localhost] => { - "ansible_facts": { - "new_usernames": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Combine users updated lists] ****************************** -ok: [localhost] => { - "ansible_facts": { - "combined_users_updated": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Debug user deduplication] ********************************* -ok: [localhost] => {} - -MSG: - -User deduplication for vnc: - - current_users_updated: ['alice', 'bob'] - - new_usernames: ['alice', 'bob'] - - combined_users_updated: ['alice', 'bob'] - - -TASK [lock_manager : Get current user secrets for list update] ***************** -ok: [localhost] => { - "ansible_facts": { - "current_user_secrets_for_list": { - "last_secret_check": "2025-07-24T15:30:30Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Update users updated list] ******************************** -ok: [localhost] => { - "ansible_facts": { - "updated_user_secrets_with_list": { - "last_secret_check": "2025-07-24T15:30:30Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Apply users updated list] ********************************* -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": true, - "completed_at": "2025-07-24T15:30:30Z" - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:30Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:30Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Get current user management data] ************************* -ok: [localhost] => { - "ansible_facts": { - "current_user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - } - }, - "changed": false -} - -TASK [lock_manager : Get current usernames list] ******************************* -ok: [localhost] => { - "ansible_facts": { - "current_usernames": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Update user management data] ****************************** -ok: [localhost] => { - "ansible_facts": { - "updated_user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - } - }, - "changed": false -} - -TASK [lock_manager : Apply user management update] ***************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": true, - "completed_at": "2025-07-24T15:30:30Z" - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:30Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:30Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Debug user management] ************************************ -ok: [localhost] => {} - -MSG: - -User management for vnc: - - current_users: ['alice', 'bob'] - - previous_users: ['alice', 'bob'] - - removed_users: [] - - -TASK [lock_manager : Check if all roles are completed] ************************* -ok: [localhost] => { - "ansible_facts": { - "all_roles_completed": false - }, - "changed": false -} - -TASK [lock_manager : Update setup status based on completion] ****************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": true, - "completed_at": "2025-07-24T15:30:30Z" - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:30Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:30Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Debug setup status update] ******************************** -ok: [localhost] => {} - -MSG: - -Setup status update: - - all_roles_completed: False - - setup_status: configuring - - completed_roles_count: 5 - - total_roles_count: 6 - - -TASK [lock_manager : Write updated lock file] ********************************** -changed: [localhost] => { - "changed": true, - "checksum": "542bc16243f6f0f5bd20bfbc7c7d0ae70dfc3965", - "dest": "/opt/vdi-setup/.vdi-lock.yaml", - "gid": 0, - "group": "root", - "md5sum": "51acde48f43e59dc50980ecbe5a85b15", - "mode": "0644", - "owner": "root", - "size": 1066, - "src": "/root/.ansible/tmp/ansible-tmp-1753371032.3061836-27078-30129676363744/source", - "state": "file", - "uid": 0 -} - -TASK [lock_manager : Debug VM metadata update] ********************************* -ok: [localhost] => {} - -MSG: - -VM metadata update info: - - hostname: vdi-test-scott-rocky-0 - - zone: us-central1-a - - lock_file_changed: True - - -TASK [lock_manager : Get lock file content as base64] ************************** -changed: [localhost] => { - "changed": true, - "cmd": "cat /opt/vdi-setup/.vdi-lock.yaml | base64 -w 0", - "delta": "0:00:00.003949", - "end": "2025-07-24 15:30:32.886270", - "rc": 0, - "start": "2025-07-24 15:30:32.882321" -} - -STDOUT: - -dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiB0cnVlCiAgICAgIGNvbXBsZXRlZF9hdDogJzIwMjUtMDctMjRUMTU6MzA6MTZaJwogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogZmFsc2UKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiB0cnVlCiAgICAgIGNvbXBsZXRlZF9hdDogJzIwMjUtMDctMjRUMTU6MzA6MzBaJwogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNTozMDozMFonCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogY29uZmlndXJpbmcKICB1c2VyX21hbmFnZW1lbnQ6CiAgICBjdXJyZW50X3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHByZXZpb3VzX3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHJlbW92ZWRfdXNlcnM6IFtdCiAgdXNlcl9zZWNyZXRzX3N0YXR1czoKICAgIGxhc3Rfc2VjcmV0X2NoZWNrOiAnMjAyNS0wNy0yNFQxNTozMDozMFonCiAgICB1c2VyX3NlY3JldHNfaGFzaDogNmE2YTAyZDNkMDk5MzNjYTI4MmFlMmIwMjMzNjM0M2M5ZDczNzAxZTQ1YmQ4MTQ3NWZiZDkyZGNmOWQyZjNiNgogICAgdXNlcnNfdXBkYXRlZDoKICAgIC0gYWxpY2UKICAgIC0gYm9iCg== - -TASK [lock_manager : Add VDI lock file content to VM metadata] ***************** -changed: [localhost] => { - "changed": true, - "cmd": [ - "gcloud", - "compute", - "instances", - "add-metadata", - "vdi-test-scott-rocky-0", - "--metadata", - "vdi-lock-content=dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiB0cnVlCiAgICAgIGNvbXBsZXRlZF9hdDogJzIwMjUtMDctMjRUMTU6MzA6MTZaJwogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogZmFsc2UKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiB0cnVlCiAgICAgIGNvbXBsZXRlZF9hdDogJzIwMjUtMDctMjRUMTU6MzA6MzBaJwogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNTozMDozMFonCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogY29uZmlndXJpbmcKICB1c2VyX21hbmFnZW1lbnQ6CiAgICBjdXJyZW50X3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHByZXZpb3VzX3VzZXJzOgogICAgLSBhbGljZQogICAgLSBib2IKICAgIHJlbW92ZWRfdXNlcnM6IFtdCiAgdXNlcl9zZWNyZXRzX3N0YXR1czoKICAgIGxhc3Rfc2VjcmV0X2NoZWNrOiAnMjAyNS0wNy0yNFQxNTozMDozMFonCiAgICB1c2VyX3NlY3JldHNfaGFzaDogNmE2YTAyZDNkMDk5MzNjYTI4MmFlMmIwMjMzNjM0M2M5ZDczNzAxZTQ1YmQ4MTQ3NWZiZDkyZGNmOWQyZjNiNgogICAgdXNlcnNfdXBkYXRlZDoKICAgIC0gYWxpY2UKICAgIC0gYm9iCg==", - "--zone=us-central1-a" - ], - "delta": "0:00:05.181207", - "end": "2025-07-24 15:30:38.265496", - "failed_when_result": false, - "rc": 0, - "start": "2025-07-24 15:30:33.084289" -} - -STDERR: - -Updated [https://www.googleapis.com/compute/v1/projects/hpc-discovery-external/zones/us-central1-a/instances/vdi-test-scott-rocky-0]. - -TASK [lock_manager : Debug lock file write] ************************************ -ok: [localhost] => {} - -MSG: - -Lock file written: True - -TASK [lock_manager : Verify lock file integrity] ******************************* -ok: [localhost] => { - "changed": false, - "cmd": [ - "python3", - "-c", - "import yaml; yaml.safe_load(open('/opt/vdi-setup/.vdi-lock.yaml'))" - ], - "delta": "0:00:00.040227", - "end": "2025-07-24 15:30:38.534489", - "rc": 0, - "start": "2025-07-24 15:30:38.494262" -} - -TASK [lock_manager : Debug lock file validation] ******************************* -ok: [localhost] => {} - -MSG: - -Lock file validation: True - -TASK [lock_manager : Check if VDI monitor service exists] ********************** -ok: [localhost] => { - "changed": false, - "stat": { - "atime": 1753370874.1548011, - "attr_flags": "", - "attributes": [], - "block_size": 4096, - "blocks": 8, - "charset": "us-ascii", - "checksum": "f6de22707f8d9248014f679a2f1930451e2bf635", - "ctime": 1753370872.6256874, - "dev": 2050, - "device_type": 0, - "executable": false, - "exists": true, - "gid": 0, - "gr_name": "root", - "inode": 285214140, - "isblk": false, - "ischr": false, - "isdir": false, - "isfifo": false, - "isgid": false, - "islnk": false, - "isreg": true, - "issock": false, - "isuid": false, - "mimetype": "text/plain", - "mode": "0644", - "mtime": 1753370872.4766762, - "nlink": 1, - "path": "/etc/systemd/system/vdi-monitor.service", - "pw_name": "root", - "readable": true, - "rgrp": true, - "roth": true, - "rusr": true, - "size": 283, - "uid": 0, - "version": "1338188335", - "wgrp": false, - "woth": false, - "writeable": true, - "wusr": true, - "xgrp": false, - "xoth": false, - "xusr": false - } -} - -TASK [lock_manager : Debug VDI monitor setup conditions] *********************** -ok: [localhost] => {} - -MSG: - -VDI monitor setup conditions: - - current_role: vnc - - vdi_monitor_service_exists: True - - lock_file_creation.changed: False - - will_setup_monitor: False - - -TASK [lock_manager : Setup VDI monitoring service (first time only)] *********** -skipping: [localhost] => { - "changed": false, - "false_condition": "not vdi_monitor_service_stat.stat.exists", - "skip_reason": "Conditional result was False" -} - -TASK [lock_manager : Fail if lock file is invalid] ***************************** -skipping: [localhost] => { - "changed": false, - "false_condition": "lock_file_validation.rc != 0", - "skip_reason": "Conditional result was False" -} - -TASK [vdi_tool : Set current role for lock manager check] ********************** -ok: [localhost] => { - "ansible_facts": { - "current_role": "vdi_tool" - }, - "changed": false -} - -TASK [lock_manager : Check if lock file exists] ******************************** -ok: [localhost] => { - "changed": false, - "stat": { - "atime": 1753371032.8846393, - "attr_flags": "", - "attributes": [], - "block_size": 4096, - "blocks": 8, - "charset": "us-ascii", - "checksum": "542bc16243f6f0f5bd20bfbc7c7d0ae70dfc3965", - "ctime": 1753371032.652622, - "dev": 2050, - "device_type": 0, - "executable": false, - "exists": true, - "gid": 0, - "gr_name": "root", - "inode": 419489989, - "isblk": false, - "ischr": false, - "isdir": false, - "isfifo": false, - "isgid": false, - "islnk": false, - "isreg": true, - "issock": false, - "isuid": false, - "mimetype": "text/plain", - "mode": "0644", - "mtime": 1753371032.49061, - "nlink": 1, - "path": "/opt/vdi-setup/.vdi-lock.yaml", - "pw_name": "root", - "readable": true, - "rgrp": true, - "roth": true, - "rusr": true, - "size": 1066, - "uid": 0, - "version": "68461029", - "wgrp": false, - "woth": false, - "writeable": true, - "wusr": true, - "xgrp": false, - "xoth": false, - "xusr": false - } -} - -TASK [lock_manager : Debug lock file existence] ******************************** -ok: [localhost] => {} - -MSG: - -Lock file exists: True - -TASK [lock_manager : Load existing lock file] ********************************** -ok: [localhost] => { - "ansible_facts": { - "vdi_setup_status": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": true, - "completed_at": "2025-07-24T15:30:30Z" - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:30Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:30Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "ansible_included_var_files": [ - "/opt/vdi-setup/.vdi-lock.yaml" - ], - "changed": false -} - -TASK [lock_manager : Debug lock file load result] ****************************** -ok: [localhost] => {} - -MSG: - -Lock file loaded successfully: True - -TASK [lock_manager : Calculate current deployment hash] ************************ -ok: [localhost] => { - "ansible_facts": { - "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab" - }, - "changed": false -} - -TASK [lock_manager : Debug deployment hash calculation] ************************ -ok: [localhost] => {} - -MSG: - -Current deployment hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab -Deployment components: - - deployment_name: vdi-test-scott - - project_id: hpc-discovery-external - - vdi_tool: guacamole - - vnc_flavor: tigervnc - - user_provision: local_users - - vdi_webapp_port: 8081 - - vdi_resolution: 1920x1080 - - vdi_users count: 2 - - -TASK [lock_manager : Calculate current user secrets hash] ********************** -ok: [localhost] => { - "ansible_facts": { - "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" - }, - "changed": false -} - -TASK [lock_manager : Debug user secrets hash calculation] ********************** -ok: [localhost] => {} - -MSG: - -Current user secrets hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 -User configuration components: - - vdi_users (blueprint): [{"username": "alice", "port": 5901}, {"username": "bob", "port": 5902, "secret_name": "a-password-for-bob"}] - - -TASK [lock_manager : Set default values for missing lock file] ***************** -skipping: [localhost] => { - "changed": false, - "false_condition": "not lock_file_stat.stat.exists", - "skip_reason": "Conditional result was False" -} - -TASK [lock_manager : Debug lock file status] *********************************** -ok: [localhost] => {} - -MSG: - -Lock file status: - - exists: True - - deployment_hash: none - - user_secrets_hash: none - - force_rerun: False - - -TASK [lock_manager : Check if lock file exists] ******************************** -ok: [localhost] => { - "ansible_facts": { - "lock_file_exists": true - }, - "changed": false -} - -TASK [lock_manager : Check if current role is completed] *********************** -ok: [localhost] => { - "ansible_facts": { - "current_role_completed": false - }, - "changed": false -} - -TASK [lock_manager : Check if deployment hash matches] ************************* -ok: [localhost] => { - "ansible_facts": { - "deployment_hash_matches": false - }, - "changed": false -} - -TASK [lock_manager : Check if force rerun is enabled] ************************** -ok: [localhost] => { - "ansible_facts": { - "force_rerun_enabled": false - }, - "changed": false -} - -TASK [lock_manager : Determine if role should run] ***************************** -ok: [localhost] => { - "ansible_facts": { - "role_should_run": true - }, - "changed": false -} - -TASK [lock_manager : Debug role should run calculation] ************************ -ok: [localhost] => {} - -MSG: - -Role should run calculation for vdi_tool: - - not lock_file_exists: False - - not current_role_completed: True - - not deployment_hash_matches: True - - force_rerun_enabled: False - - role_should_run: True - - -TASK [lock_manager : Debug role execution decision] **************************** -ok: [localhost] => {} - -MSG: - -Role execution decision for vdi_tool: - - current_role: vdi_tool - - ansible_role_name: lock_manager - - role_should_run: True - - lock_file_exists: True - - current_role_completed: False - - deployment_hash_matches: False - - force_rerun_enabled: False - - -TASK [lock_manager : Check if user configuration has changed] ****************** -ok: [localhost] => { - "ansible_facts": { - "user_config_changed": true - }, - "changed": false -} - -TASK [lock_manager : Set user secrets changed flag (will be updated by secret_manager role)] *** -ok: [localhost] => { - "ansible_facts": { - "user_secrets_changed": true - }, - "changed": false -} - -TASK [lock_manager : Debug user secrets change detection] ********************** -ok: [localhost] => {} - -MSG: - -User configuration change detection: - - user_config_changed: True - - user_secrets_changed: True - - stored_hash: none - - current_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 - - -TASK [lock_manager : Set lock check result] ************************************ -ok: [localhost] => { - "ansible_facts": { - "lock_check_result": { - "current_deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "current_user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "existing_deployment_hash": "none", - "existing_user_secrets_hash": "none", - "force_rerun": false, - "role_should_run": true, - "user_secrets_changed": true - } - }, - "changed": false -} - -TASK [lock_manager : Debug final lock check result] **************************** -ok: [localhost] => {} - -MSG: - -Final lock check result: - - role_should_run: True - - deployment_hash_changed: True - - user_secrets_changed: True - - force_rerun: False - - -TASK [vdi_tool : Set role_should_run from lock check result] ******************* -ok: [localhost] => { - "ansible_facts": { - "role_should_run": true - }, - "changed": false -} - -TASK [vdi_tool : Skip vdi_tool tasks if role should not run] ******************* -skipping: [localhost] => { - "false_condition": "not role_should_run" -} - -TASK [vdi_tool : Install Guacamole] ******************************************** -included: /tmp/vdi/roles/vdi_tool/tasks/guacamole.yaml for localhost - -TASK [vdi_tool : Ensure PostgreSQL data & init directories exist] ************** -changed: [localhost] => (item=/opt/guacamole-db/data) => { - "ansible_loop_var": "item", - "changed": true, - "gid": 0, - "group": "root", - "item": "/opt/guacamole-db/data", - "mode": "0755", - "owner": "root", - "path": "/opt/guacamole-db/data", - "size": 6, - "state": "directory", - "uid": 0 -} -changed: [localhost] => (item=/opt/guacamole-db/initdb) => { - "ansible_loop_var": "item", - "changed": true, - "gid": 0, - "group": "root", - "item": "/opt/guacamole-db/initdb", - "mode": "0755", - "owner": "root", - "path": "/opt/guacamole-db/initdb", - "size": 6, - "state": "directory", - "uid": 0 -} - -TASK [vdi_tool : Check Docker service status] ********************************** -ok: [localhost] => { - "ansible_facts": { - "services": { - "NetworkManager-dispatcher.service": { - "name": "NetworkManager-dispatcher.service", - "source": "systemd", - "state": "inactive", - "status": "enabled" - }, - "NetworkManager-wait-online.service": { - "name": "NetworkManager-wait-online.service", - "source": "systemd", - "state": "stopped", - "status": "enabled" - }, - "NetworkManager.service": { - "name": "NetworkManager.service", - "source": "systemd", - "state": "running", - "status": "enabled" - }, - "accounts-daemon.service": { - "name": "accounts-daemon.service", - "source": "systemd", - "state": "running", - "status": "enabled" - }, - "acpid.service": { - "name": "acpid.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "apt-daily.service": { - "name": "apt-daily.service", - "source": "systemd", - "state": "stopped", - "status": "not-found" - }, - "arp-ethers.service": { - "name": "arp-ethers.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "atd.service": { - "name": "atd.service", - "source": "systemd", - "state": "running", - "status": "enabled" - }, - "auditd.service": { - "name": "auditd.service", - "source": "systemd", - "state": "running", - "status": "enabled" - }, - "auth-rpcgss-module.service": { - "name": "auth-rpcgss-module.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "auto-cpufreq.service": { - "name": "auto-cpufreq.service", - "source": "systemd", - "state": "stopped", - "status": "not-found" - }, - "autovt@.service": { - "name": "autovt@.service", - "source": "systemd", - "state": "unknown", - "status": "enabled" - }, - "bluetooth.service": { - "name": "bluetooth.service", - "source": "systemd", - "state": "inactive", - "status": "enabled" - }, - "bolt.service": { - "name": "bolt.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "btattach-bcm@.service": { - "name": "btattach-bcm@.service", - "source": "systemd", - "state": "unknown", - "status": "static" - }, - "canberra-system-bootup.service": { - "name": "canberra-system-bootup.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "canberra-system-shutdown-reboot.service": { - "name": "canberra-system-shutdown-reboot.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "canberra-system-shutdown.service": { - "name": "canberra-system-shutdown.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "chrony-dnssrv@.service": { - "name": "chrony-dnssrv@.service", - "source": "systemd", - "state": "unknown", - "status": "static" - }, - "chrony-wait.service": { - "name": "chrony-wait.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "chronyd.service": { - "name": "chronyd.service", - "source": "systemd", - "state": "running", - "status": "enabled" - }, - "colord.service": { - "name": "colord.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "console-getty.service": { - "name": "console-getty.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "container-getty@.service": { - "name": "container-getty@.service", - "source": "systemd", - "state": "unknown", - "status": "static" - }, - "containerd.service": { - "name": "containerd.service", - "source": "systemd", - "state": "running", - "status": "disabled" - }, - "cpupower.service": { - "name": "cpupower.service", - "source": "systemd", - "state": "stopped", - "status": "disabled" - }, - "crond.service": { - "name": "crond.service", - "source": "systemd", - "state": "running", - "status": "enabled" - }, - "daos_agent.service": { - "name": "daos_agent.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "dbus-org.bluez.service": { - "name": "dbus-org.bluez.service", - "source": "systemd", - "state": "inactive", - "status": "enabled" - }, - "dbus-org.freedesktop.hostname1.service": { - "name": "dbus-org.freedesktop.hostname1.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "dbus-org.freedesktop.locale1.service": { - "name": "dbus-org.freedesktop.locale1.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "dbus-org.freedesktop.login1.service": { - "name": "dbus-org.freedesktop.login1.service", - "source": "systemd", - "state": "active", - "status": "static" - }, - "dbus-org.freedesktop.nm-dispatcher.service": { - "name": "dbus-org.freedesktop.nm-dispatcher.service", - "source": "systemd", - "state": "inactive", - "status": "enabled" - }, - "dbus-org.freedesktop.portable1.service": { - "name": "dbus-org.freedesktop.portable1.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "dbus-org.freedesktop.timedate1.service": { - "name": "dbus-org.freedesktop.timedate1.service", - "source": "systemd", - "state": "inactive", - "status": "enabled" - }, - "dbus.service": { - "name": "dbus.service", - "source": "systemd", - "state": "running", - "status": "static" - }, - "debug-shell.service": { - "name": "debug-shell.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "display-manager.service": { - "name": "display-manager.service", - "source": "systemd", - "state": "inactive", - "status": "enabled" - }, - "dnf-automatic-download.service": { - "name": "dnf-automatic-download.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "dnf-automatic-install.service": { - "name": "dnf-automatic-install.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "dnf-automatic-notifyonly.service": { - "name": "dnf-automatic-notifyonly.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "dnf-automatic.service": { - "name": "dnf-automatic.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "dnf-makecache.service": { - "name": "dnf-makecache.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "dnf-system-upgrade-cleanup.service": { - "name": "dnf-system-upgrade-cleanup.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "dnf-system-upgrade.service": { - "name": "dnf-system-upgrade.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "docker.service": { - "name": "docker.service", - "source": "systemd", - "state": "running", - "status": "enabled" - }, - "dracut-cmdline.service": { - "name": "dracut-cmdline.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "dracut-initqueue.service": { - "name": "dracut-initqueue.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "dracut-mount.service": { - "name": "dracut-mount.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "dracut-pre-mount.service": { - "name": "dracut-pre-mount.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "dracut-pre-pivot.service": { - "name": "dracut-pre-pivot.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "dracut-pre-trigger.service": { - "name": "dracut-pre-trigger.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "dracut-pre-udev.service": { - "name": "dracut-pre-udev.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "dracut-shutdown-onfailure.service": { - "name": "dracut-shutdown-onfailure.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "dracut-shutdown.service": { - "name": "dracut-shutdown.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "ebtables.service": { - "name": "ebtables.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "emergency.service": { - "name": "emergency.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "firewalld.service": { - "name": "firewalld.service", - "source": "systemd", - "state": "stopped", - "status": "masked" - }, - "flatpak-system-helper.service": { - "name": "flatpak-system-helper.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "fstrim.service": { - "name": "fstrim.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "fwupd-offline-update.service": { - "name": "fwupd-offline-update.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "fwupd-refresh.service": { - "name": "fwupd-refresh.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "fwupd.service": { - "name": "fwupd.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "gce-workload-cert-refresh.service": { - "name": "gce-workload-cert-refresh.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "gdm.service": { - "name": "gdm.service", - "source": "systemd", - "state": "stopped", - "status": "enabled" - }, - "geoclue.service": { - "name": "geoclue.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "getty@.service": { - "name": "getty@.service", - "source": "systemd", - "state": "unknown", - "status": "enabled" - }, - "getty@tty1.service": { - "name": "getty@tty1.service", - "source": "systemd", - "state": "running", - "status": "active" - }, - "google-disk-expand.service": { - "name": "google-disk-expand.service", - "source": "systemd", - "state": "stopped", - "status": "enabled" - }, - "google-guest-agent-manager.service": { - "name": "google-guest-agent-manager.service", - "source": "systemd", - "state": "running", - "status": "enabled" - }, - "google-guest-agent.service": { - "name": "google-guest-agent.service", - "source": "systemd", - "state": "running", - "status": "enabled" - }, - "google-guest-compat-manager.service": { - "name": "google-guest-compat-manager.service", - "source": "systemd", - "state": "running", - "status": "enabled" - }, - "google-hpc-firstrun.service": { - "name": "google-hpc-firstrun.service", - "source": "systemd", - "state": "inactive", - "status": "enabled" - }, - "google-hpc-multiqueue.service": { - "name": "google-hpc-multiqueue.service", - "source": "systemd", - "state": "stopped", - "status": "enabled" - }, - "google-osconfig-agent.service": { - "name": "google-osconfig-agent.service", - "source": "systemd", - "state": "running", - "status": "enabled" - }, - "google-oslogin-cache.service": { - "name": "google-oslogin-cache.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "google-shutdown-scripts.service": { - "name": "google-shutdown-scripts.service", - "source": "systemd", - "state": "stopped", - "status": "enabled" - }, - "google-startup-scripts.service": { - "name": "google-startup-scripts.service", - "source": "systemd", - "state": "stopped", - "status": "enabled" - }, - "grub-boot-indeterminate.service": { - "name": "grub-boot-indeterminate.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "gssproxy.service": { - "name": "gssproxy.service", - "source": "systemd", - "state": "running", - "status": "disabled" - }, - "halt-local.service": { - "name": "halt-local.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "hwloc-dump-hwdata.service": { - "name": "hwloc-dump-hwdata.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "iio-sensor-proxy.service": { - "name": "iio-sensor-proxy.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "import-state.service": { - "name": "import-state.service", - "source": "systemd", - "state": "stopped", - "status": "enabled" - }, - "initrd-cleanup.service": { - "name": "initrd-cleanup.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "initrd-parse-etc.service": { - "name": "initrd-parse-etc.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "initrd-switch-root.service": { - "name": "initrd-switch-root.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "initrd-udevadm-cleanup-db.service": { - "name": "initrd-udevadm-cleanup-db.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "iprdump.service": { - "name": "iprdump.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "iprinit.service": { - "name": "iprinit.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "iprupdate.service": { - "name": "iprupdate.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "kdump.service": { - "name": "kdump.service", - "source": "systemd", - "state": "stopped", - "status": "enabled" - }, - "kmod-static-nodes.service": { - "name": "kmod-static-nodes.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "kvm_stat.service": { - "name": "kvm_stat.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "ldconfig.service": { - "name": "ldconfig.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "loadmodules.service": { - "name": "loadmodules.service", - "source": "systemd", - "state": "stopped", - "status": "enabled" - }, - "man-db-cache-update.service": { - "name": "man-db-cache-update.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "man-db-restart-cache-update.service": { - "name": "man-db-restart-cache-update.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "mdadm-grow-continue@.service": { - "name": "mdadm-grow-continue@.service", - "source": "systemd", - "state": "unknown", - "status": "static" - }, - "mdadm-last-resort@.service": { - "name": "mdadm-last-resort@.service", - "source": "systemd", - "state": "unknown", - "status": "static" - }, - "mdcheck_continue.service": { - "name": "mdcheck_continue.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "mdcheck_start.service": { - "name": "mdcheck_start.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "mdmon@.service": { - "name": "mdmon@.service", - "source": "systemd", - "state": "unknown", - "status": "static" - }, - "mdmonitor-oneshot.service": { - "name": "mdmonitor-oneshot.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "mdmonitor.service": { - "name": "mdmonitor.service", - "source": "systemd", - "state": "stopped", - "status": "enabled" - }, - "messagebus.service": { - "name": "messagebus.service", - "source": "systemd", - "state": "active", - "status": "static" - }, - "modprobe@efi_pstore.service": { - "name": "modprobe@efi_pstore.service", - "source": "systemd", - "state": "stopped", - "status": "not-found" - }, - "mount-localssd-raid.service": { - "name": "mount-localssd-raid.service", - "source": "systemd", - "state": "stopped", - "status": "not-found" - }, - "network.service": { - "name": "network.service", - "source": "systemd", - "state": "stopped", - "status": "not-found" - }, - "networking.service": { - "name": "networking.service", - "source": "systemd", - "state": "stopped", - "status": "not-found" - }, - "nfs-blkmap.service": { - "name": "nfs-blkmap.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "nfs-convert.service": { - "name": "nfs-convert.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "nfs-idmapd.service": { - "name": "nfs-idmapd.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "nfs-mountd.service": { - "name": "nfs-mountd.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "nfs-server.service": { - "name": "nfs-server.service", - "source": "systemd", - "state": "stopped", - "status": "disabled" - }, - "nfs-utils.service": { - "name": "nfs-utils.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "nfsdcld.service": { - "name": "nfsdcld.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "nftables.service": { - "name": "nftables.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "nis-domainname.service": { - "name": "nis-domainname.service", - "source": "systemd", - "state": "stopped", - "status": "enabled" - }, - "ntpd.service": { - "name": "ntpd.service", - "source": "systemd", - "state": "stopped", - "status": "not-found" - }, - "ntpdate.service": { - "name": "ntpdate.service", - "source": "systemd", - "state": "stopped", - "status": "not-found" - }, - "nvmefc-boot-connections.service": { - "name": "nvmefc-boot-connections.service", - "source": "systemd", - "state": "inactive", - "status": "enabled" - }, - "nvmf-autoconnect.service": { - "name": "nvmf-autoconnect.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "nvmf-connect@.service": { - "name": "nvmf-connect@.service", - "source": "systemd", - "state": "unknown", - "status": "static" - }, - "pesign.service": { - "name": "pesign.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "plymouth-halt.service": { - "name": "plymouth-halt.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "plymouth-kexec.service": { - "name": "plymouth-kexec.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "plymouth-poweroff.service": { - "name": "plymouth-poweroff.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "plymouth-quit-wait.service": { - "name": "plymouth-quit-wait.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "plymouth-quit.service": { - "name": "plymouth-quit.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "plymouth-read-write.service": { - "name": "plymouth-read-write.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "plymouth-reboot.service": { - "name": "plymouth-reboot.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "plymouth-start.service": { - "name": "plymouth-start.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "plymouth-switch-root-initramfs.service": { - "name": "plymouth-switch-root-initramfs.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "plymouth-switch-root.service": { - "name": "plymouth-switch-root.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "polkit.service": { - "name": "polkit.service", - "source": "systemd", - "state": "running", - "status": "static" - }, - "power-profiles-daemon.service": { - "name": "power-profiles-daemon.service", - "source": "systemd", - "state": "stopped", - "status": "not-found" - }, - "qemu-guest-agent.service": { - "name": "qemu-guest-agent.service", - "source": "systemd", - "state": "inactive", - "status": "enabled" - }, - "quotaon.service": { - "name": "quotaon.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "rc-local.service": { - "name": "rc-local.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "rdisc.service": { - "name": "rdisc.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "rdma-load-modules@.service": { - "name": "rdma-load-modules@.service", - "source": "systemd", - "state": "unknown", - "status": "static" - }, - "rdma-ndd.service": { - "name": "rdma-ndd.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "rescue.service": { - "name": "rescue.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "rngd.service": { - "name": "rngd.service", - "source": "systemd", - "state": "running", - "status": "enabled" - }, - "rpc-gssd.service": { - "name": "rpc-gssd.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "rpc-statd-notify.service": { - "name": "rpc-statd-notify.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "rpc-statd.service": { - "name": "rpc-statd.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "rpcbind.service": { - "name": "rpcbind.service", - "source": "systemd", - "state": "running", - "status": "enabled" - }, - "rsyslog.service": { - "name": "rsyslog.service", - "source": "systemd", - "state": "running", - "status": "enabled" - }, - "rtkit-daemon.service": { - "name": "rtkit-daemon.service", - "source": "systemd", - "state": "running", - "status": "enabled" - }, - "selinux-autorelabel-mark.service": { - "name": "selinux-autorelabel-mark.service", - "source": "systemd", - "state": "stopped", - "status": "enabled" - }, - "selinux-autorelabel.service": { - "name": "selinux-autorelabel.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "serial-getty@.service": { - "name": "serial-getty@.service", - "source": "systemd", - "state": "unknown", - "status": "indirect" - }, - "serial-getty@ttyS0.service": { - "name": "serial-getty@ttyS0.service", - "source": "systemd", - "state": "running", - "status": "active" - }, - "serial-getty@ttyS1.service": { - "name": "serial-getty@ttyS1.service", - "source": "systemd", - "state": "running", - "status": "active" - }, - "sntp.service": { - "name": "sntp.service", - "source": "systemd", - "state": "stopped", - "status": "not-found" - }, - "sshd-keygen@.service": { - "name": "sshd-keygen@.service", - "source": "systemd", - "state": "unknown", - "status": "disabled" - }, - "sshd-keygen@ecdsa.service": { - "name": "sshd-keygen@ecdsa.service", - "source": "systemd", - "state": "stopped", - "status": "inactive" - }, - "sshd-keygen@ed25519.service": { - "name": "sshd-keygen@ed25519.service", - "source": "systemd", - "state": "stopped", - "status": "inactive" - }, - "sshd-keygen@rsa.service": { - "name": "sshd-keygen@rsa.service", - "source": "systemd", - "state": "stopped", - "status": "inactive" - }, - "sshd.service": { - "name": "sshd.service", - "source": "systemd", - "state": "running", - "status": "enabled" - }, - "sshd@.service": { - "name": "sshd@.service", - "source": "systemd", - "state": "unknown", - "status": "static" - }, - "sssd-autofs.service": { - "name": "sssd-autofs.service", - "source": "systemd", - "state": "inactive", - "status": "indirect" - }, - "sssd-kcm.service": { - "name": "sssd-kcm.service", - "source": "systemd", - "state": "stopped", - "status": "indirect" - }, - "sssd-nss.service": { - "name": "sssd-nss.service", - "source": "systemd", - "state": "inactive", - "status": "indirect" - }, - "sssd-pac.service": { - "name": "sssd-pac.service", - "source": "systemd", - "state": "inactive", - "status": "indirect" - }, - "sssd-pam.service": { - "name": "sssd-pam.service", - "source": "systemd", - "state": "inactive", - "status": "indirect" - }, - "sssd-ssh.service": { - "name": "sssd-ssh.service", - "source": "systemd", - "state": "inactive", - "status": "indirect" - }, - "sssd-sudo.service": { - "name": "sssd-sudo.service", - "source": "systemd", - "state": "inactive", - "status": "indirect" - }, - "sssd.service": { - "name": "sssd.service", - "source": "systemd", - "state": "stopped", - "status": "enabled" - }, - "switcheroo-control.service": { - "name": "switcheroo-control.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "syslog.service": { - "name": "syslog.service", - "source": "systemd", - "state": "stopped", - "status": "not-found" - }, - "system-update-cleanup.service": { - "name": "system-update-cleanup.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "systemd-ask-password-console.service": { - "name": "systemd-ask-password-console.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-ask-password-plymouth.service": { - "name": "systemd-ask-password-plymouth.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-ask-password-wall.service": { - "name": "systemd-ask-password-wall.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-backlight@.service": { - "name": "systemd-backlight@.service", - "source": "systemd", - "state": "unknown", - "status": "static" - }, - "systemd-binfmt.service": { - "name": "systemd-binfmt.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-coredump@.service": { - "name": "systemd-coredump@.service", - "source": "systemd", - "state": "unknown", - "status": "static" - }, - "systemd-exit.service": { - "name": "systemd-exit.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "systemd-firstboot.service": { - "name": "systemd-firstboot.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-fsck-root.service": { - "name": "systemd-fsck-root.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-fsck@.service": { - "name": "systemd-fsck@.service", - "source": "systemd", - "state": "unknown", - "status": "static" - }, - "systemd-fsck@dev-disk-by\\x2duuid-580C\\x2d02F4.service": { - "name": "systemd-fsck@dev-disk-by\\x2duuid-580C\\x2d02F4.service", - "source": "systemd", - "state": "stopped", - "status": "active" - }, - "systemd-halt.service": { - "name": "systemd-halt.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "systemd-hibernate-resume@.service": { - "name": "systemd-hibernate-resume@.service", - "source": "systemd", - "state": "unknown", - "status": "static" - }, - "systemd-hibernate.service": { - "name": "systemd-hibernate.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "systemd-hostnamed.service": { - "name": "systemd-hostnamed.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "systemd-hwdb-update.service": { - "name": "systemd-hwdb-update.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-hybrid-sleep.service": { - "name": "systemd-hybrid-sleep.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "systemd-initctl.service": { - "name": "systemd-initctl.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-journal-catalog-update.service": { - "name": "systemd-journal-catalog-update.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-journal-flush.service": { - "name": "systemd-journal-flush.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-journald.service": { - "name": "systemd-journald.service", - "source": "systemd", - "state": "running", - "status": "static" - }, - "systemd-kexec.service": { - "name": "systemd-kexec.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "systemd-localed.service": { - "name": "systemd-localed.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "systemd-logind.service": { - "name": "systemd-logind.service", - "source": "systemd", - "state": "running", - "status": "static" - }, - "systemd-machine-id-commit.service": { - "name": "systemd-machine-id-commit.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-modules-load.service": { - "name": "systemd-modules-load.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-networkd.service": { - "name": "systemd-networkd.service", - "source": "systemd", - "state": "stopped", - "status": "not-found" - }, - "systemd-portabled.service": { - "name": "systemd-portabled.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "systemd-poweroff.service": { - "name": "systemd-poweroff.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "systemd-pstore.service": { - "name": "systemd-pstore.service", - "source": "systemd", - "state": "stopped", - "status": "enabled" - }, - "systemd-quotacheck.service": { - "name": "systemd-quotacheck.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-random-seed.service": { - "name": "systemd-random-seed.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-reboot.service": { - "name": "systemd-reboot.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "systemd-remount-fs.service": { - "name": "systemd-remount-fs.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-resolved.service": { - "name": "systemd-resolved.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "systemd-rfkill.service": { - "name": "systemd-rfkill.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "systemd-suspend-then-hibernate.service": { - "name": "systemd-suspend-then-hibernate.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "systemd-suspend.service": { - "name": "systemd-suspend.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "systemd-sysctl.service": { - "name": "systemd-sysctl.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-sysusers.service": { - "name": "systemd-sysusers.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-timedated.service": { - "name": "systemd-timedated.service", - "source": "systemd", - "state": "inactive", - "status": "masked" - }, - "systemd-timesyncd.service": { - "name": "systemd-timesyncd.service", - "source": "systemd", - "state": "stopped", - "status": "not-found" - }, - "systemd-tmpfiles-clean.service": { - "name": "systemd-tmpfiles-clean.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-tmpfiles-setup-dev.service": { - "name": "systemd-tmpfiles-setup-dev.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-tmpfiles-setup.service": { - "name": "systemd-tmpfiles-setup.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-udev-settle.service": { - "name": "systemd-udev-settle.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "systemd-udev-trigger.service": { - "name": "systemd-udev-trigger.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-udevd.service": { - "name": "systemd-udevd.service", - "source": "systemd", - "state": "running", - "status": "static" - }, - "systemd-update-done.service": { - "name": "systemd-update-done.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-update-utmp-runlevel.service": { - "name": "systemd-update-utmp-runlevel.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-update-utmp.service": { - "name": "systemd-update-utmp.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-user-sessions.service": { - "name": "systemd-user-sessions.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-vconsole-setup.service": { - "name": "systemd-vconsole-setup.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "systemd-volatile-root.service": { - "name": "systemd-volatile-root.service", - "source": "systemd", - "state": "inactive", - "status": "static" - }, - "tcsd.service": { - "name": "tcsd.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "teamd@.service": { - "name": "teamd@.service", - "source": "systemd", - "state": "unknown", - "status": "static" - }, - "timedatex.service": { - "name": "timedatex.service", - "source": "systemd", - "state": "inactive", - "status": "enabled" - }, - "tlp.service": { - "name": "tlp.service", - "source": "systemd", - "state": "stopped", - "status": "not-found" - }, - "tuned.service": { - "name": "tuned.service", - "source": "systemd", - "state": "running", - "status": "enabled" - }, - "udisks2.service": { - "name": "udisks2.service", - "source": "systemd", - "state": "stopped", - "status": "enabled" - }, - "unbound-anchor.service": { - "name": "unbound-anchor.service", - "source": "systemd", - "state": "stopped", - "status": "static" - }, - "upower.service": { - "name": "upower.service", - "source": "systemd", - "state": "running", - "status": "disabled" - }, - "user-runtime-dir@.service": { - "name": "user-runtime-dir@.service", - "source": "systemd", - "state": "unknown", - "status": "static" - }, - "user-runtime-dir@1000.service": { - "name": "user-runtime-dir@1000.service", - "source": "systemd", - "state": "stopped", - "status": "active" - }, - "user-runtime-dir@1001.service": { - "name": "user-runtime-dir@1001.service", - "source": "systemd", - "state": "stopped", - "status": "active" - }, - "user-runtime-dir@87404603.service": { - "name": "user-runtime-dir@87404603.service", - "source": "systemd", - "state": "stopped", - "status": "active" - }, - "user@.service": { - "name": "user@.service", - "source": "systemd", - "state": "unknown", - "status": "static" - }, - "user@1000.service": { - "name": "user@1000.service", - "source": "systemd", - "state": "running", - "status": "active" - }, - "user@1001.service": { - "name": "user@1001.service", - "source": "systemd", - "state": "running", - "status": "active" - }, - "user@87404603.service": { - "name": "user@87404603.service", - "source": "systemd", - "state": "running", - "status": "active" - }, - "vdi-monitor.service": { - "name": "vdi-monitor.service", - "source": "systemd", - "state": "running", - "status": "enabled" - }, - "vncserver@.service": { - "name": "vncserver@.service", - "source": "systemd", - "state": "unknown", - "status": "indirect" - }, - "vncserver@:1.service": { - "name": "vncserver@:1.service", - "source": "systemd", - "state": "running", - "status": "active" - }, - "vncserver@:2.service": { - "name": "vncserver@:2.service", - "source": "systemd", - "state": "running", - "status": "active" - }, - "wacom-inputattach@.service": { - "name": "wacom-inputattach@.service", - "source": "systemd", - "state": "unknown", - "status": "static" - }, - "wpa_supplicant.service": { - "name": "wpa_supplicant.service", - "source": "systemd", - "state": "inactive", - "status": "disabled" - }, - "xvnc@.service": { - "name": "xvnc@.service", - "source": "systemd", - "state": "unknown", - "status": "static" - }, - "ypbind.service": { - "name": "ypbind.service", - "source": "systemd", - "state": "stopped", - "status": "not-found" - }, - "yppasswdd.service": { - "name": "yppasswdd.service", - "source": "systemd", - "state": "stopped", - "status": "not-found" - }, - "ypserv.service": { - "name": "ypserv.service", - "source": "systemd", - "state": "stopped", - "status": "not-found" - }, - "ypxfrd.service": { - "name": "ypxfrd.service", - "source": "systemd", - "state": "stopped", - "status": "not-found" - } - } - }, - "changed": false -} - -TASK [vdi_tool : Ensure Docker service is running] ***************************** -ok: [localhost] => { - "changed": false, - "enabled": true, - "name": "docker", - "state": "started", - "status": { - "ActiveEnterTimestamp": "Thu 2025-07-24 15:27:29 UTC", - "ActiveEnterTimestampMonotonic": "259862320", - "ActiveExitTimestampMonotonic": "0", - "ActiveState": "active", - "After": "network-online.target containerd.service sysinit.target systemd-journald.socket basic.target docker.socket firewalld.service time-set.target system.slice nss-lookup.target mount-localssd-raid.service", - "AllowIsolate": "no", - "AllowedCPUs": "", - "AllowedMemoryNodes": "", - "AmbientCapabilities": "", - "AssertResult": "yes", - "AssertTimestamp": "Thu 2025-07-24 15:27:26 UTC", - "AssertTimestampMonotonic": "257032900", - "Before": "shutdown.target multi-user.target", - "BlockIOAccounting": "no", - "BlockIOWeight": "[not set]", - "CPUAccounting": "no", - "CPUAffinity": "", - "CPUAffinityFromNUMA": "no", - "CPUQuotaPerSecUSec": "infinity", - "CPUQuotaPeriodUSec": "infinity", - "CPUSchedulingPolicy": "0", - "CPUSchedulingPriority": "0", - "CPUSchedulingResetOnFork": "no", - "CPUShares": "[not set]", - "CPUUsageNSec": "[not set]", - "CPUWeight": "[not set]", - "CacheDirectoryMode": "0755", - "CanFreeze": "yes", - "CanIsolate": "no", - "CanReload": "yes", - "CanStart": "yes", - "CanStop": "yes", - "CapabilityBoundingSet": "cap_chown cap_dac_override cap_dac_read_search cap_fowner cap_fsetid cap_kill cap_setgid cap_setuid cap_setpcap cap_linux_immutable cap_net_bind_service cap_net_broadcast cap_net_admin cap_net_raw cap_ipc_lock cap_ipc_owner cap_sys_module cap_sys_rawio cap_sys_chroot cap_sys_ptrace cap_sys_pacct cap_sys_admin cap_sys_boot cap_sys_nice cap_sys_resource cap_sys_time cap_sys_tty_config cap_mknod cap_lease cap_audit_write cap_audit_control cap_setfcap cap_mac_override cap_mac_admin cap_syslog cap_wake_alarm cap_block_suspend cap_audit_read cap_perfmon cap_bpf", - "CollectMode": "inactive", - "ConditionResult": "yes", - "ConditionTimestamp": "Thu 2025-07-24 15:27:26 UTC", - "ConditionTimestampMonotonic": "257032900", - "ConfigurationDirectoryMode": "0755", - "Conflicts": "shutdown.target", - "ControlGroup": "/system.slice/docker.service", - "ControlPID": "0", - "DefaultDependencies": "yes", - "DefaultMemoryLow": "0", - "DefaultMemoryMin": "0", - "Delegate": "yes", - "DelegateControllers": "cpu cpuacct cpuset io blkio memory devices pids", - "Description": "Docker Application Container Engine", - "DevicePolicy": "auto", - "Documentation": "https://docs.docker.com", - "DropInPaths": "/etc/systemd/system/docker.service.d/data-root.conf", - "DynamicUser": "no", - "EffectiveCPUs": "", - "EffectiveMemoryNodes": "", - "ExecMainCode": "0", - "ExecMainExitTimestampMonotonic": "0", - "ExecMainPID": "5274", - "ExecMainStartTimestamp": "Thu 2025-07-24 15:27:26 UTC", - "ExecMainStartTimestampMonotonic": "257033635", - "ExecMainStatus": "0", - "ExecReload": "{ path=/bin/kill ; argv[]=/bin/kill -s HUP $MAINPID ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", - "ExecStart": "{ path=/usr/bin/dockerd ; argv[]=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", - "FailureAction": "none", - "FileDescriptorStoreMax": "0", - "FragmentPath": "/usr/lib/systemd/system/docker.service", - "FreezerState": "running", - "GID": "[not set]", - "GuessMainPID": "yes", - "IOAccounting": "no", - "IOSchedulingClass": "0", - "IOSchedulingPriority": "0", - "IOWeight": "[not set]", - "IPAccounting": "no", - "IPEgressBytes": "18446744073709551615", - "IPEgressPackets": "18446744073709551615", - "IPIngressBytes": "18446744073709551615", - "IPIngressPackets": "18446744073709551615", - "Id": "docker.service", - "IgnoreOnIsolate": "no", - "IgnoreSIGPIPE": "yes", - "InactiveEnterTimestampMonotonic": "0", - "InactiveExitTimestamp": "Thu 2025-07-24 15:27:26 UTC", - "InactiveExitTimestampMonotonic": "257033661", - "InvocationID": "2c563fb7673f41a09f94812ff7281e0d", - "JobRunningTimeoutUSec": "infinity", - "JobTimeoutAction": "none", - "JobTimeoutUSec": "infinity", - "KeyringMode": "private", - "KillMode": "process", - "KillSignal": "15", - "LimitAS": "infinity", - "LimitASSoft": "infinity", - "LimitCORE": "infinity", - "LimitCORESoft": "infinity", - "LimitCPU": "infinity", - "LimitCPUSoft": "infinity", - "LimitDATA": "infinity", - "LimitDATASoft": "infinity", - "LimitFSIZE": "infinity", - "LimitFSIZESoft": "infinity", - "LimitLOCKS": "infinity", - "LimitLOCKSSoft": "infinity", - "LimitMEMLOCK": "65536", - "LimitMEMLOCKSoft": "65536", - "LimitMSGQUEUE": "819200", - "LimitMSGQUEUESoft": "819200", - "LimitNICE": "0", - "LimitNICESoft": "0", - "LimitNOFILE": "262144", - "LimitNOFILESoft": "1024", - "LimitNPROC": "infinity", - "LimitNPROCSoft": "infinity", - "LimitRSS": "infinity", - "LimitRSSSoft": "infinity", - "LimitRTPRIO": "0", - "LimitRTPRIOSoft": "0", - "LimitRTTIME": "infinity", - "LimitRTTIMESoft": "infinity", - "LimitSIGPENDING": "127383", - "LimitSIGPENDINGSoft": "127383", - "LimitSTACK": "infinity", - "LimitSTACKSoft": "8388608", - "LoadState": "loaded", - "LockPersonality": "no", - "LogLevelMax": "-1", - "LogRateLimitBurst": "0", - "LogRateLimitIntervalUSec": "0", - "LogsDirectoryMode": "0755", - "MainPID": "5274", - "MemoryAccounting": "yes", - "MemoryCurrent": "31453184", - "MemoryDenyWriteExecute": "no", - "MemoryHigh": "infinity", - "MemoryLimit": "infinity", - "MemoryLow": "0", - "MemoryMax": "infinity", - "MemoryMin": "0", - "MemorySwapMax": "infinity", - "MountAPIVFS": "no", - "MountFlags": "", - "NFileDescriptorStore": "0", - "NRestarts": "0", - "NUMAMask": "", - "NUMAPolicy": "n/a", - "Names": "docker.service", - "NeedDaemonReload": "no", - "Nice": "0", - "NoNewPrivileges": "no", - "NonBlocking": "no", - "NotifyAccess": "main", - "OOMScoreAdjust": "-500", - "OnFailureJobMode": "replace", - "PermissionsStartOnly": "no", - "Perpetual": "no", - "PrivateDevices": "no", - "PrivateMounts": "no", - "PrivateNetwork": "no", - "PrivateTmp": "no", - "PrivateUsers": "no", - "ProtectControlGroups": "no", - "ProtectHome": "no", - "ProtectKernelModules": "no", - "ProtectKernelTunables": "no", - "ProtectSystem": "no", - "RefuseManualStart": "no", - "RefuseManualStop": "no", - "RemainAfterExit": "no", - "RemoveIPC": "no", - "Requires": "system.slice sysinit.target docker.socket", - "Restart": "always", - "RestartUSec": "2s", - "RestrictNamespaces": "no", - "RestrictRealtime": "no", - "RestrictSUIDSGID": "no", - "Result": "success", - "RootDirectoryStartOnly": "no", - "RuntimeDirectoryMode": "0755", - "RuntimeDirectoryPreserve": "no", - "RuntimeMaxUSec": "infinity", - "SameProcessGroup": "no", - "SecureBits": "0", - "SendSIGHUP": "no", - "SendSIGKILL": "yes", - "Slice": "system.slice", - "StandardError": "inherit", - "StandardInput": "null", - "StandardInputData": "", - "StandardOutput": "journal", - "StartLimitAction": "none", - "StartLimitBurst": "3", - "StartLimitIntervalUSec": "1min", - "StartupBlockIOWeight": "[not set]", - "StartupCPUShares": "[not set]", - "StartupCPUWeight": "[not set]", - "StartupIOWeight": "[not set]", - "StateChangeTimestamp": "Thu 2025-07-24 15:27:29 UTC", - "StateChangeTimestampMonotonic": "259862320", - "StateDirectoryMode": "0755", - "StatusErrno": "0", - "StopWhenUnneeded": "no", - "SubState": "running", - "SuccessAction": "none", - "SyslogFacility": "3", - "SyslogLevel": "6", - "SyslogLevelPrefix": "yes", - "SyslogPriority": "30", - "SystemCallErrorNumber": "0", - "TTYReset": "no", - "TTYVHangup": "no", - "TTYVTDisallocate": "no", - "TasksAccounting": "yes", - "TasksCurrent": "10", - "TasksMax": "infinity", - "TimeoutStartUSec": "infinity", - "TimeoutStopUSec": "1min 30s", - "TimerSlackNSec": "50000", - "Transient": "no", - "TriggeredBy": "docker.socket", - "Type": "notify", - "UID": "[not set]", - "UMask": "0022", - "UnitFilePreset": "disabled", - "UnitFileState": "enabled", - "UtmpMode": "init", - "WantedBy": "multi-user.target", - "Wants": "network-online.target containerd.service", - "WatchdogTimestamp": "Thu 2025-07-24 15:27:29 UTC", - "WatchdogTimestampMonotonic": "259862318", - "WatchdogUSec": "0" - } -} - -TASK [vdi_tool : Wait for Docker daemon to be ready] *************************** -ok: [localhost] => { - "changed": false, - "elapsed": 0, - "gid": 988, - "group": "docker", - "match_groupdict": {}, - "match_groups": [], - "mode": "0660", - "owner": "root", - "path": "/var/run/docker.sock", - "port": null, - "search_regex": null, - "size": 0, - "state": "file", - "uid": 0 -} - -TASK [vdi_tool : Test Docker connectivity] ************************************* -changed: [localhost] => { - "attempts": 1, - "changed": true, - "cmd": [ - "docker", - "version" - ], - "delta": "0:00:00.019833", - "end": "2025-07-24 15:31:01.979111", - "rc": 0, - "start": "2025-07-24 15:31:01.959278" -} - -STDOUT: - -Client: Docker Engine - Community - Version: 28.3.2 - API version: 1.51 - Go version: go1.24.5 - Git commit: 578ccf6 - Built: Wed Jul 9 16:15:41 2025 - OS/Arch: linux/amd64 - Context: default - -Server: Docker Engine - Community - Engine: - Version: 28.3.2 - API version: 1.51 (minimum version 1.24) - Go version: go1.24.5 - Git commit: e77ff99 - Built: Wed Jul 9 16:14:23 2025 - OS/Arch: linux/amd64 - Experimental: false - containerd: - Version: 1.7.27 - GitCommit: 05044ec0a9a75232cad458027ca83437aae3f4da - runc: - Version: 1.2.5 - GitCommit: v1.2.5-0-g59923ef - docker-init: - Version: 0.19.0 - GitCommit: de40ad0 - -TASK [vdi_tool : Enable pgcrypto extension for initdb] ************************* -changed: [localhost] => { - "changed": true, - "checksum": "a127ec70504ae55424b5bfdba85a53ce3fe2e710", - "dest": "/opt/guacamole-db/initdb/00-enable-pgcrypto.sql", - "gid": 0, - "group": "root", - "md5sum": "5759c18ba008c3af3b174a99dfedc20e", - "mode": "0644", - "owner": "root", - "size": 82, - "src": "/root/.ansible/tmp/ansible-tmp-1753371062.0273864-27523-222276405449246/source", - "state": "file", - "uid": 0 -} - -TASK [vdi_tool : Template per-user Guacamole user SQL] ************************* -changed: [localhost] => (item={'username': 'alice', 'port': 5901, 'password': 'xc61q4jU3mUCo84M', 'vncserver_password': '4JB1YHUQ', 'display_number': 1}) => { - "ansible_loop_var": "item", - "changed": true, - "checksum": "11e358d1c93056ca1f3197b5a7aa118878d69048", - "dest": "/opt/guacamole-db/initdb/02-alice_user.sql", - "gid": 0, - "group": "root", - "item": { - "display_number": 1, - "password": "xc61q4jU3mUCo84M", - "port": 5901, - "username": "alice", - "vncserver_password": "4JB1YHUQ" - }, - "md5sum": "2203ca51f69a089f76a44664596df631", - "mode": "0644", - "owner": "root", - "size": 734, - "src": "/root/.ansible/tmp/ansible-tmp-1753371062.378293-27545-63386938039965/source", - "state": "file", - "uid": 0 -} -changed: [localhost] => (item={'username': 'bob', 'port': 5902, 'secret_name': 'a-password-for-bob', 'password': 'B0bp4ssw0rd123', 'vncserver_password': '7DmxvFp2', 'display_number': 2}) => { - "ansible_loop_var": "item", - "changed": true, - "checksum": "9eabaa254800e4e435f8c70b918a4a6b428b6fbd", - "dest": "/opt/guacamole-db/initdb/02-bob_user.sql", - "gid": 0, - "group": "root", - "item": { - "display_number": 2, - "password": "B0bp4ssw0rd123", - "port": 5902, - "secret_name": "a-password-for-bob", - "username": "bob", - "vncserver_password": "7DmxvFp2" - }, - "md5sum": "73275e670b160e53c0acca30b52ae218", - "mode": "0644", - "owner": "root", - "size": 726, - "src": "/root/.ansible/tmp/ansible-tmp-1753371062.7223604-27545-271943085306166/source", - "state": "file", - "uid": 0 -} - -TASK [vdi_tool : Template per-user SQL for Guacamole connections] ************** -changed: [localhost] => (item={'username': 'alice', 'port': 5901, 'password': 'xc61q4jU3mUCo84M', 'vncserver_password': '4JB1YHUQ', 'display_number': 1}) => { - "ansible_loop_var": "item", - "changed": true, - "checksum": "a2f7d7fd0c1e596f84464e301312c00e0fa12a11", - "dest": "/opt/guacamole-db/initdb/03-alice_connection.sql", - "gid": 0, - "group": "root", - "item": { - "display_number": 1, - "password": "xc61q4jU3mUCo84M", - "port": 5901, - "username": "alice", - "vncserver_password": "4JB1YHUQ" - }, - "md5sum": "7df88ce1ea5f854682df295bfe2e301f", - "mode": "0644", - "owner": "root", - "size": 6583, - "src": "/root/.ansible/tmp/ansible-tmp-1753371063.081999-27587-195959994230920/source", - "state": "file", - "uid": 0 -} -changed: [localhost] => (item={'username': 'bob', 'port': 5902, 'secret_name': 'a-password-for-bob', 'password': 'B0bp4ssw0rd123', 'vncserver_password': '7DmxvFp2', 'display_number': 2}) => { - "ansible_loop_var": "item", - "changed": true, - "checksum": "291f5f0588de9e18befd6b849591ee5eab224333", - "dest": "/opt/guacamole-db/initdb/03-bob_connection.sql", - "gid": 0, - "group": "root", - "item": { - "display_number": 2, - "password": "B0bp4ssw0rd123", - "port": 5902, - "secret_name": "a-password-for-bob", - "username": "bob", - "vncserver_password": "7DmxvFp2" - }, - "md5sum": "1ae0fd1017d8d23d1acd289524cc7b5c", - "mode": "0644", - "owner": "root", - "size": 6547, - "src": "/root/.ansible/tmp/ansible-tmp-1753371063.4521995-27587-78729102064992/source", - "state": "file", - "uid": 0 -} - -TASK [vdi_tool : Template user cleanup SQL for removed users] ****************** -changed: [localhost] => { - "changed": true, - "checksum": "eed1f453966cbb3acf4a3266ca3b5998416a3706", - "dest": "/opt/guacamole-db/initdb/04-user_cleanup.sql", - "gid": 0, - "group": "root", - "md5sum": "b582c47aac06c808a4bb374e7dbad148", - "mode": "0644", - "owner": "root", - "size": 3166, - "src": "/root/.ansible/tmp/ansible-tmp-1753371063.8163276-27629-260203219362437/source", - "state": "file", - "uid": 0 -} - -TASK [vdi_tool : Generate combined initdb.sql] ********************************* -changed: [localhost] => { - "changed": true, - "cmd": "docker run --rm -v /opt/guacamole-db/initdb:/opt/guacamole-db/initdb guacamole/guacamole:latest /opt/guacamole/bin/initdb.sh --postgresql > /opt/guacamole-db/initdb/01-initdb.sql\n", - "delta": "0:00:12.451850", - "end": "2025-07-24 15:31:16.794061", - "rc": 0, - "start": "2025-07-24 15:31:04.342211" -} - -STDERR: - -Unable to find image 'guacamole/guacamole:latest' locally -latest: Pulling from guacamole/guacamole -32f112e3802c: Pulling fs layer -ea8f5ca39c1a: Pulling fs layer -a4f9b8cc7730: Pulling fs layer -4e45d9955da7: Pulling fs layer -59425f3d7529: Pulling fs layer -a875e361a596: Pulling fs layer -4f4fb700ef54: Pulling fs layer -cdbfaaae41bb: Pulling fs layer -f5cb881fc6db: Pulling fs layer -de7057472cfc: Pulling fs layer -7c43b83d031a: Pulling fs layer -8a4673b53887: Pulling fs layer -8630345c18ca: Pulling fs layer -a875e361a596: Waiting -4f4fb700ef54: Waiting -cdbfaaae41bb: Waiting -f5cb881fc6db: Waiting -de7057472cfc: Waiting -7c43b83d031a: Waiting -8a4673b53887: Waiting -8630345c18ca: Waiting -4e45d9955da7: Waiting -59425f3d7529: Waiting -ea8f5ca39c1a: Verifying Checksum -ea8f5ca39c1a: Download complete -32f112e3802c: Verifying Checksum -32f112e3802c: Download complete -4e45d9955da7: Download complete -59425f3d7529: Verifying Checksum -59425f3d7529: Download complete -a875e361a596: Verifying Checksum -a875e361a596: Download complete -4f4fb700ef54: Verifying Checksum -4f4fb700ef54: Download complete -cdbfaaae41bb: Verifying Checksum -cdbfaaae41bb: Download complete -de7057472cfc: Verifying Checksum -de7057472cfc: Download complete -f5cb881fc6db: Verifying Checksum -f5cb881fc6db: Download complete -32f112e3802c: Pull complete -8a4673b53887: Download complete -8630345c18ca: Download complete -ea8f5ca39c1a: Pull complete -a4f9b8cc7730: Verifying Checksum -a4f9b8cc7730: Download complete -7c43b83d031a: Verifying Checksum -7c43b83d031a: Download complete -a4f9b8cc7730: Pull complete -4e45d9955da7: Pull complete -59425f3d7529: Pull complete -a875e361a596: Pull complete -4f4fb700ef54: Pull complete -cdbfaaae41bb: Pull complete -f5cb881fc6db: Pull complete -de7057472cfc: Pull complete -7c43b83d031a: Pull complete -8a4673b53887: Pull complete -8630345c18ca: Pull complete -Digest: sha256:4cec231c1fe46367e1e3da80a12f0f59c48aebde9e6a97cdaf2a8fcf92376b4a -Status: Downloaded newer image for guacamole/guacamole:latest - -TASK [vdi_tool : Update guacadmin password hash in 01-initdb.sql] ************** -changed: [localhost] => { - "changed": true, - "rc": 0 -} - -MSG: - -1 replacements made - -TASK [vdi_tool : Remove salt from guacadmin in 01-initdb.sql] ****************** -changed: [localhost] => { - "changed": true, - "rc": 0 -} - -MSG: - -1 replacements made - -TASK [vdi_tool : Pull required Docker images] ********************************** -changed: [localhost] => (item=postgres:latest) => { - "ansible_loop_var": "item", - "changed": true, - "cmd": [ - "docker", - "pull", - "postgres:latest" - ], - "delta": "0:00:05.944704", - "end": "2025-07-24 15:31:23.396321", - "item": "postgres:latest", - "rc": 0, - "start": "2025-07-24 15:31:17.451617" -} - -STDOUT: - -latest: Pulling from library/postgres -59e22667830b: Pulling fs layer -c2922dd5c76b: Pulling fs layer -28b0f0abf5b3: Pulling fs layer -a623957f847d: Pulling fs layer -bfabbcdba989: Pulling fs layer -0e5bba37029a: Pulling fs layer -1ed9ff0b5160: Pulling fs layer -06b2fd76987c: Pulling fs layer -9e482b60495e: Pulling fs layer -4a43c57a903d: Pulling fs layer -dcce1c0b58a8: Pulling fs layer -38072b29d55e: Pulling fs layer -d98e5494f240: Pulling fs layer -017403a0b5e1: Pulling fs layer -0e5bba37029a: Waiting -1ed9ff0b5160: Waiting -06b2fd76987c: Waiting -9e482b60495e: Waiting -4a43c57a903d: Waiting -dcce1c0b58a8: Waiting -38072b29d55e: Waiting -d98e5494f240: Waiting -017403a0b5e1: Waiting -a623957f847d: Waiting -bfabbcdba989: Waiting -c2922dd5c76b: Verifying Checksum -c2922dd5c76b: Download complete -28b0f0abf5b3: Verifying Checksum -28b0f0abf5b3: Download complete -a623957f847d: Verifying Checksum -a623957f847d: Download complete -59e22667830b: Verifying Checksum -59e22667830b: Download complete -bfabbcdba989: Verifying Checksum -bfabbcdba989: Download complete -0e5bba37029a: Verifying Checksum -0e5bba37029a: Download complete -1ed9ff0b5160: Verifying Checksum -1ed9ff0b5160: Download complete -06b2fd76987c: Verifying Checksum -06b2fd76987c: Download complete -4a43c57a903d: Verifying Checksum -4a43c57a903d: Download complete -dcce1c0b58a8: Download complete -38072b29d55e: Download complete -d98e5494f240: Verifying Checksum -d98e5494f240: Download complete -59e22667830b: Pull complete -c2922dd5c76b: Pull complete -017403a0b5e1: Verifying Checksum -017403a0b5e1: Download complete -28b0f0abf5b3: Pull complete -a623957f847d: Pull complete -bfabbcdba989: Pull complete -0e5bba37029a: Pull complete -1ed9ff0b5160: Pull complete -06b2fd76987c: Pull complete -9e482b60495e: Verifying Checksum -9e482b60495e: Download complete -9e482b60495e: Pull complete -4a43c57a903d: Pull complete -dcce1c0b58a8: Pull complete -38072b29d55e: Pull complete -d98e5494f240: Pull complete -017403a0b5e1: Pull complete -Digest: sha256:4d89c904835259bc58876520e56267ca07a4ebd6a027f7814bbbf91b50d685be -Status: Downloaded newer image for postgres:latest -docker.io/library/postgres:latest -changed: [localhost] => (item=guacamole/guacd:latest) => { - "ansible_loop_var": "item", - "changed": true, - "cmd": [ - "docker", - "pull", - "guacamole/guacd:latest" - ], - "delta": "0:00:04.829341", - "end": "2025-07-24 15:31:28.396327", - "item": "guacamole/guacd:latest", - "rc": 0, - "start": "2025-07-24 15:31:23.566986" -} - -STDOUT: - -latest: Pulling from guacamole/guacd -44cf07d57ee4: Pulling fs layer -2c7959c3f8de: Pulling fs layer -d977524fd227: Pulling fs layer -26483f1a0b5c: Pulling fs layer -e2df05816112: Pulling fs layer -596f1cdf21db: Pulling fs layer -26483f1a0b5c: Waiting -596f1cdf21db: Waiting -e2df05816112: Waiting -2c7959c3f8de: Download complete -44cf07d57ee4: Download complete -44cf07d57ee4: Pull complete -26483f1a0b5c: Verifying Checksum -26483f1a0b5c: Download complete -e2df05816112: Verifying Checksum -e2df05816112: Download complete -2c7959c3f8de: Pull complete -596f1cdf21db: Verifying Checksum -596f1cdf21db: Download complete -d977524fd227: Download complete -d977524fd227: Pull complete -26483f1a0b5c: Pull complete -e2df05816112: Pull complete -596f1cdf21db: Pull complete -Digest: sha256:f70aa3356d1ae6ba7cff608309684c1736c7ae7b2b2e201e8e29281f2591227f -Status: Downloaded newer image for guacamole/guacd:latest -docker.io/guacamole/guacd:latest -changed: [localhost] => (item=guacamole/guacamole:latest) => { - "ansible_loop_var": "item", - "changed": true, - "cmd": [ - "docker", - "pull", - "guacamole/guacamole:latest" - ], - "delta": "0:00:00.359246", - "end": "2025-07-24 15:31:28.927638", - "item": "guacamole/guacamole:latest", - "rc": 0, - "start": "2025-07-24 15:31:28.568392" -} - -STDOUT: - -latest: Pulling from guacamole/guacamole -Digest: sha256:4cec231c1fe46367e1e3da80a12f0f59c48aebde9e6a97cdaf2a8fcf92376b4a -Status: Image is up to date for guacamole/guacamole:latest -docker.io/guacamole/guacamole:latest - -TASK [vdi_tool : Create a dedicated Docker network] **************************** -changed: [localhost] => { - "changed": true, - "cmd": [ - "docker", - "network", - "create", - "guac_net" - ], - "delta": "0:00:00.062475", - "end": "2025-07-24 15:31:29.181269", - "failed_when_result": false, - "rc": 0, - "start": "2025-07-24 15:31:29.118794" -} - -STDOUT: - -e0275c25416d70f991c99ebe7fb8ee8b8ca67ff902b5ea23f21f58461ec48bff - -TASK [vdi_tool : Start Guacamole PostgreSQL container] ************************* -changed: [localhost] => { - "changed": true, - "cmd": [ - "docker", - "run", - "-d", - "--name", - "guac_db", - "--network", - "guac_net", - "-e", - "POSTGRES_USER=guacamole_db", - "-e", - "POSTGRES_PASSWORD=H95L0yeLji7pzjDCOuemJDzEwoASP7Nq", - "-e", - "POSTGRES_DB=guacamole_db", - "-v", - "/opt/guacamole-db/data:/var/lib/postgresql/data", - "-v", - "/opt/guacamole-db/initdb:/docker-entrypoint-initdb.d", - "--restart", - "always", - "postgres:latest" - ], - "delta": "0:00:07.462040", - "end": "2025-07-24 15:31:36.839313", - "failed_when_result": false, - "rc": 0, - "start": "2025-07-24 15:31:29.377273" -} - -STDOUT: - -7f665322894de32870a3505d6da6ae46f83ea5d228a347e7ea1a71240c15015e - -TASK [vdi_tool : Wait for PostgreSQL container to be ready] ******************** -fatal: [localhost]: FAILED! => { - "changed": false, - "elapsed": 60 -} - -MSG: - -Timeout when waiting for 127.0.0.1:5432 -...ignoring - -TASK [vdi_tool : Wait for PostgreSQL via Docker exec] ************************** -changed: [localhost] => { - "attempts": 1, - "changed": true, - "cmd": [ - "docker", - "exec", - "guac_db", - "pg_isready", - "-U", - "guacamole_db" - ], - "delta": "0:00:00.088759", - "end": "2025-07-24 15:32:37.343504", - "rc": 0, - "start": "2025-07-24 15:32:37.254745" -} - -STDOUT: - -/var/run/postgresql:5432 - accepting connections - -TASK [vdi_tool : Check if database needs re-initialization due to user changes] *** -ok: [localhost] => { - "changed": false, - "cmd": [ - "docker", - "exec", - "guac_db", - "psql", - "-U", - "guacamole_db", - "-d", - "guacamole_db", - "-c", - "SELECT COUNT(*) FROM guacamole_entity WHERE type='USER' AND name != 'guacadmin';" - ], - "delta": "0:00:00.083631", - "end": "2025-07-24 15:32:37.618923", - "rc": 0, - "start": "2025-07-24 15:32:37.535292" -} - -STDOUT: - - count -------- - 2 -(1 row) - -TASK [vdi_tool : Debug user configuration status] ****************************** -ok: [localhost] => {} - -MSG: - -Database user check: count -------- - 2 -(1 row) -Current users in config: 2 -Lock file exists: True -User secrets hash changed: True -User secrets hash (stored): none -User secrets hash (current): 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 - - -TASK [vdi_tool : Check if user configuration has changed] ********************** -ok: [localhost] => { - "ansible_facts": { - "users_changed": true - }, - "changed": false -} - -TASK [vdi_tool : Debug user change detection] ********************************** -ok: [localhost] => {} - -MSG: - -Users changed: True - -TASK [vdi_tool : Stop Guacamole containers] ************************************ -fatal: [localhost]: FAILED! => { - "changed": true, - "cmd": [ - "docker", - "stop", - "guac_app", - "guacd" - ], - "delta": "0:00:00.013660", - "end": "2025-07-24 15:32:37.908714", - "rc": 1, - "start": "2025-07-24 15:32:37.895054" -} - -STDERR: - -Error response from daemon: No such container: guac_app -Error response from daemon: No such container: guacd - - -MSG: - -non-zero return code -...ignoring - -TASK [vdi_tool : Remove Guacamole containers] ********************************** -fatal: [localhost]: FAILED! => { - "changed": true, - "cmd": [ - "docker", - "rm", - "guac_app", - "guacd" - ], - "delta": "0:00:00.012838", - "end": "2025-07-24 15:32:38.111949", - "rc": 1, - "start": "2025-07-24 15:32:38.099111" -} - -STDERR: - -Error response from daemon: No such container: guac_app -Error response from daemon: No such container: guacd - - -MSG: - -non-zero return code -...ignoring - -TASK [vdi_tool : Stop and remove PostgreSQL container] ************************* -changed: [localhost] => { - "changed": true, - "cmd": [ - "docker", - "stop", - "guac_db" - ], - "delta": "0:00:00.134068", - "end": "2025-07-24 15:32:38.439413", - "rc": 0, - "start": "2025-07-24 15:32:38.305345" -} - -STDOUT: - -guac_db - -TASK [vdi_tool : Remove PostgreSQL container] ********************************** -changed: [localhost] => { - "changed": true, - "cmd": [ - "docker", - "rm", - "guac_db" - ], - "delta": "0:00:00.018178", - "end": "2025-07-24 15:32:38.655370", - "rc": 0, - "start": "2025-07-24 15:32:38.637192" -} - -STDOUT: - -guac_db - -TASK [vdi_tool : Backup existing database data] ******************************** -fatal: [localhost]: FAILED! => { - "changed": true, - "cmd": [ - "tar", - "-czf", - "/opt/guacamole-db/backup-$(date", - "+%Y%m%d-%H%M%S).tar.gz", - "-C", - "/opt/guacamole-db", - "data/" - ], - "delta": "0:00:00.927459", - "end": "2025-07-24 15:32:39.776351", - "rc": 2, - "start": "2025-07-24 15:32:38.848892" -} - -STDERR: - -tar: +%Y%m%d-%H%M%S).tar.gz: Cannot stat: No such file or directory -tar: Exiting with failure status due to previous errors - - -MSG: - -non-zero return code -...ignoring - -TASK [vdi_tool : Remove existing database data] ******************************** -changed: [localhost] => { - "changed": true, - "cmd": [ - "rm", - "-rf", - "/opt/guacamole-db/data/*" - ], - "delta": "0:00:00.002506", - "end": "2025-07-24 15:32:39.973183", - "rc": 0, - "start": "2025-07-24 15:32:39.970677" -} - -TASK [vdi_tool : Start Guacamole PostgreSQL container (re-initialized)] ******** -changed: [localhost] => { - "changed": true, - "cmd": [ - "docker", - "run", - "-d", - "--name", - "guac_db", - "--network", - "guac_net", - "-e", - "POSTGRES_USER=guacamole_db", - "-e", - "POSTGRES_PASSWORD=H95L0yeLji7pzjDCOuemJDzEwoASP7Nq", - "-e", - "POSTGRES_DB=guacamole_db", - "-v", - "/opt/guacamole-db/data:/var/lib/postgresql/data", - "-v", - "/opt/guacamole-db/initdb:/docker-entrypoint-initdb.d", - "--restart", - "always", - "postgres:latest" - ], - "delta": "0:00:00.214960", - "end": "2025-07-24 15:32:40.389033", - "failed_when_result": false, - "rc": 0, - "start": "2025-07-24 15:32:40.174073" -} - -STDOUT: - -f5711132145daa089efb4d0a763437ab81d2f13cdde1644b2472b45869dd4c81 - -TASK [vdi_tool : Wait for PostgreSQL to be ready after re-initialization] ****** -changed: [localhost] => { - "attempts": 1, - "changed": true, - "cmd": [ - "docker", - "exec", - "guac_db", - "pg_isready", - "-U", - "guacamole_db" - ], - "delta": "0:00:00.084157", - "end": "2025-07-24 15:32:40.671774", - "rc": 0, - "start": "2025-07-24 15:32:40.587617" -} - -STDOUT: - -/var/run/postgresql:5432 - accepting connections - -TASK [vdi_tool : Start guacd container] **************************************** -changed: [localhost] => { - "changed": true, - "cmd": [ - "docker", - "run", - "-d", - "--name", - "guacd", - "--network", - "guac_net", - "-p", - "4822:4822", - "--restart", - "always", - "guacamole/guacd:latest" - ], - "delta": "0:00:00.199585", - "end": "2025-07-24 15:32:41.064927", - "failed_when_result": false, - "rc": 0, - "start": "2025-07-24 15:32:40.865342" -} - -STDOUT: - -809f2c58fa7ebb5c36b56d14a529a6261f8fe3e30e1b6993d36b92d9cf905269 - -TASK [vdi_tool : Start Guacamole webapp container] ***************************** -changed: [localhost] => { - "changed": true, - "cmd": [ - "docker", - "run", - "-d", - "--name", - "guac_app", - "--network", - "guac_net", - "-p", - "8081:8080", - "-e", - "POSTGRES_HOSTNAME=guac_db", - "-e", - "POSTGRES_PORT=5432", - "-e", - "POSTGRES_DATABASE=guacamole_db", - "-e", - "POSTGRES_USER=guacamole_db", - "-e", - "POSTGRES_PASSWORD=H95L0yeLji7pzjDCOuemJDzEwoASP7Nq", - "-e", - "POSTGRES_AUTO_CREATE_ACCOUNTS=true", - "-e", - "GUACD_HOSTNAME=guacd", - "-e", - "GUACD_PORT=4822", - "--restart", - "always", - "guacamole/guacamole:latest" - ], - "delta": "0:00:00.203470", - "end": "2025-07-24 15:32:41.469743", - "failed_when_result": false, - "rc": 0, - "start": "2025-07-24 15:32:41.266273" -} - -STDOUT: - -671a01e330bf70d61b6a5aa26d573704e651f02a8b292abd5a8fb656bd885675 - -TASK [vdi_tool : Debug container status before waiting for Guacamole] ********** -ok: [localhost] => { - "changed": false, - "cmd": [ - "docker", - "ps" - ], - "delta": "0:00:00.015784", - "end": "2025-07-24 15:32:41.680190", - "rc": 0, - "start": "2025-07-24 15:32:41.664406" -} - -STDOUT: - -CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -671a01e330bf guacamole/guacamole:latest "/opt/guacamole/bin/ā€¦" Less than a second ago Up Less than a second 0.0.0.0:8081->8080/tcp, [::]:8081->8080/tcp guac_app -809f2c58fa7e guacamole/guacd:latest "/opt/guacamole/entrā€¦" 1 second ago Up Less than a second (health: starting) 0.0.0.0:4822->4822/tcp, [::]:4822->4822/tcp guacd -f5711132145d postgres:latest "docker-entrypoint.sā€¦" 1 second ago Up 1 second 5432/tcp guac_db - -TASK [vdi_tool : Show container status] **************************************** -ok: [localhost] => {} - -MSG: - -CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -671a01e330bf guacamole/guacamole:latest "/opt/guacamole/bin/ā€¦" Less than a second ago Up Less than a second 0.0.0.0:8081->8080/tcp, [::]:8081->8080/tcp guac_app -809f2c58fa7e guacamole/guacd:latest "/opt/guacamole/entrā€¦" 1 second ago Up Less than a second (health: starting) 0.0.0.0:4822->4822/tcp, [::]:4822->4822/tcp guacd -f5711132145d postgres:latest "docker-entrypoint.sā€¦" 1 second ago Up 1 second 5432/tcp guac_db - -TASK [vdi_tool : Wait for Guacamole HTTP endpoint] ***************************** -ok: [localhost] => { - "changed": false, - "elapsed": 2, - "match_groupdict": {}, - "match_groups": [], - "path": null, - "port": 8081, - "search_regex": null, - "state": "started" -} - -TASK [vdi_tool : Check Guacamole UI is up] ************************************* -ok: [localhost] => { - "accept_ranges": "bytes", - "attempts": 1, - "cache_control": "no-cache", - "changed": false, - "connection": "close", - "content_length": "2811", - "content_type": "text/html", - "cookies": {}, - "cookies_string": "", - "date": "Thu, 24 Jul 2025 15:32:45 GMT", - "elapsed": 0, - "etag": "W/\"2811-1753318742000\"", - "last_modified": "Thu, 24 Jul 2025 00:59:02 GMT", - "pragma": "no-cache", - "redirected": false, - "status": 200, - "url": "http://localhost:8081/guacamole/" -} - -MSG: - -OK (2811 bytes) - -TASK [vdi_tool : Obtain Guacamole API token to verify login] ******************* -ok: [localhost] => { - "attempts": 1, - "changed": false, - "connection": "close", - "content": "{\"authToken\":\"0D1A88B239CAB8F2BB454E77803481AA313179A99217A10E06963CAFFC219737\",\"username\":\"guacadmin\",\"dataSource\":\"postgresql\",\"availableDataSources\":[\"postgresql\",\"postgresql-shared\"]}", - "content_length": "187", - "content_type": "application/json", - "cookies": {}, - "cookies_string": "", - "date": "Thu, 24 Jul 2025 15:32:45 GMT", - "elapsed": 0, - "json": { - "authToken": "0D1A88B239CAB8F2BB454E77803481AA313179A99217A10E06963CAFFC219737", - "availableDataSources": [ - "postgresql", - "postgresql-shared" - ], - "dataSource": "postgresql", - "username": "guacadmin" - }, - "redirected": false, - "status": 200, - "url": "http://localhost:8081/guacamole/api/tokens" -} - -MSG: - -OK (187 bytes) - -TASK [vdi_tool : Remove bootstrap SQL files unless debug is enabled] *********** -skipping: [localhost] => { - "changed": false, - "false_condition": "not debug | default(false)", - "skip_reason": "Conditional result was False" -} - -TASK [vdi_tool : Install NoMachine] ******************************************** -skipping: [localhost] => { - "changed": false, - "false_condition": "vdi_tool | lower == 'nomachine'", - "skip_reason": "Conditional result was False" -} - -TASK [vdi_tool : Install Workspot] ********************************************* -skipping: [localhost] => { - "changed": false, - "false_condition": "vdi_tool | lower == 'workspot'", - "skip_reason": "Conditional result was False" -} - -TASK [vdi_tool : Set current role for lock manager completion] ***************** -ok: [localhost] => { - "ansible_facts": { - "current_role": "vdi_tool", - "role_completed": true - }, - "changed": false -} - -TASK [lock_manager : Debug create lock operation] ****************************** -ok: [localhost] => {} - -MSG: - -Creating/updating lock file for role: vdi_tool - -TASK [lock_manager : Check if lock file exists] ******************************** -ok: [localhost] => { - "changed": false, - "stat": { - "atime": 1753371032.8846393, - "attr_flags": "", - "attributes": [], - "block_size": 4096, - "blocks": 8, - "charset": "us-ascii", - "checksum": "542bc16243f6f0f5bd20bfbc7c7d0ae70dfc3965", - "ctime": 1753371032.652622, - "dev": 2050, - "device_type": 0, - "executable": false, - "exists": true, - "gid": 0, - "gr_name": "root", - "inode": 419489989, - "isblk": false, - "ischr": false, - "isdir": false, - "isfifo": false, - "isgid": false, - "islnk": false, - "isreg": true, - "issock": false, - "isuid": false, - "mimetype": "text/plain", - "mode": "0644", - "mtime": 1753371032.49061, - "nlink": 1, - "path": "/opt/vdi-setup/.vdi-lock.yaml", - "pw_name": "root", - "readable": true, - "rgrp": true, - "roth": true, - "rusr": true, - "size": 1066, - "uid": 0, - "version": "68461029", - "wgrp": false, - "woth": false, - "writeable": true, - "wusr": true, - "xgrp": false, - "xoth": false, - "xusr": false - } -} - -TASK [lock_manager : Debug lock file existence] ******************************** -ok: [localhost] => {} - -MSG: - -Lock file exists: True - -TASK [lock_manager : Create initial lock file structure] *********************** -skipping: [localhost] => { - "changed": false, - "false_condition": "not lock_file_stat.stat.exists", - "skip_reason": "Conditional result was False" -} - -TASK [lock_manager : Debug lock file creation] ********************************* -skipping: [localhost] => { - "false_condition": "debug | default(false) and not lock_file_stat.stat.exists" -} - -TASK [lock_manager : Load existing lock file data] ***************************** -ok: [localhost] => { - "ansible_facts": { - "vdi_setup_status": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": true, - "completed_at": "2025-07-24T15:30:30Z" - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:30Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:30Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "ansible_included_var_files": [ - "/opt/vdi-setup/.vdi-lock.yaml" - ], - "changed": false -} - -TASK [lock_manager : Get current lock data] ************************************ -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": true, - "completed_at": "2025-07-24T15:30:30Z" - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:30Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:30Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Update deployment hash] *********************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": true, - "completed_at": "2025-07-24T15:30:30Z" - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:30:30Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:30Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Get current timestamp] ************************************ -changed: [localhost] => { - "changed": true, - "cmd": [ - "date", - "-u", - "+%Y-%m-%dT%H:%M:%SZ" - ], - "delta": "0:00:00.002389", - "end": "2025-07-24 15:32:46.698118", - "rc": 0, - "start": "2025-07-24 15:32:46.695729" -} - -STDOUT: - -2025-07-24T15:32:46Z - -TASK [lock_manager : Update last updated timestamp] **************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": true, - "completed_at": "2025-07-24T15:30:30Z" - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:32:46Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:30:30Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Get current user secrets status] ************************** -ok: [localhost] => { - "ansible_facts": { - "current_user_secrets": { - "last_secret_check": "2025-07-24T15:30:30Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Create user secrets update] ******************************* -ok: [localhost] => { - "ansible_facts": { - "user_secrets_update": { - "last_secret_check": "2025-07-24T15:32:46Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6" - } - }, - "changed": false -} - -TASK [lock_manager : Update user secrets hash] ********************************* -ok: [localhost] => { - "ansible_facts": { - "updated_user_secrets": { - "last_secret_check": "2025-07-24T15:32:46Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Apply user secrets update] ******************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": true, - "completed_at": "2025-07-24T15:30:30Z" - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:32:46Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:32:46Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Create role status entry] ********************************* -ok: [localhost] => { - "ansible_facts": { - "role_status_entry": { - "completed": true, - "completed_at": "2025-07-24T15:32:46Z" - } - }, - "changed": false -} - -TASK [lock_manager : Debug role status entry] ********************************** -ok: [localhost] => {} - -MSG: - -Role status entry for vdi_tool: - - completed: True - - completed_at: 2025-07-24T15:32:46Z - - -TASK [lock_manager : Debug current lock data before role update] *************** -ok: [localhost] => {} - -MSG: - -Current lock data before updating vdi_tool: - - current_lock_data: completed_roles: - base_os: - completed: true - completed_at: '2025-07-24T15:29:50Z' - lock_manager: - completed: true - completed_at: '2025-07-24T15:27:44Z' - secret_manager: - completed: true - completed_at: '2025-07-24T15:30:03Z' - user_provision: - completed: true - completed_at: '2025-07-24T15:30:16Z' - vdi_tool: - completed: false - vnc: - completed: true - completed_at: '2025-07-24T15:30:30Z' -created_at: '2025-07-24T15:27:41Z' -deployment_hash: 4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab -deployment_name: vdi-test-scott -force_rerun: false -last_updated: '2025-07-24T15:32:46Z' -lock_version: '1.0' -setup_status: configuring -user_management: - current_users: - - alice - - bob - previous_users: - - alice - - bob - removed_users: [] -user_secrets_status: - last_secret_check: '2025-07-24T15:32:46Z' - user_secrets_hash: 6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6 - users_updated: - - alice - - bob - - - current_completed_roles: base_os: - completed: true - completed_at: '2025-07-24T15:29:50Z' -lock_manager: - completed: true - completed_at: '2025-07-24T15:27:44Z' -secret_manager: - completed: true - completed_at: '2025-07-24T15:30:03Z' -user_provision: - completed: true - completed_at: '2025-07-24T15:30:16Z' -vdi_tool: - completed: false -vnc: - completed: true - completed_at: '2025-07-24T15:30:30Z' - - - role_completed: True - - current_role: vdi_tool - - -TASK [lock_manager : Get current completed roles] ****************************** -ok: [localhost] => { - "ansible_facts": { - "current_completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": false - }, - "vnc": { - "completed": true, - "completed_at": "2025-07-24T15:30:30Z" - } - } - }, - "changed": false -} - -TASK [lock_manager : Update completed roles] *********************************** -ok: [localhost] => { - "ansible_facts": { - "updated_completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": true, - "completed_at": "2025-07-24T15:32:46Z" - }, - "vnc": { - "completed": true, - "completed_at": "2025-07-24T15:30:30Z" - } - } - }, - "changed": false -} - -TASK [lock_manager : Update lock data with completed roles] ******************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": true, - "completed_at": "2025-07-24T15:32:46Z" - }, - "vnc": { - "completed": true, - "completed_at": "2025-07-24T15:30:30Z" - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:32:46Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:32:46Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Debug updated lock data after role update] **************** -ok: [localhost] => {} - -MSG: - -Updated lock data after updating vdi_tool: - - current_role: vdi_tool - - role_completed: True - - completed_roles: {'base_os': {'completed': True, 'completed_at': '2025-07-24T15:29:50Z'}, 'lock_manager': {'completed': True, 'completed_at': '2025-07-24T15:27:44Z'}, 'secret_manager': {'completed': True, 'completed_at': '2025-07-24T15:30:03Z'}, 'user_provision': {'completed': True, 'completed_at': '2025-07-24T15:30:16Z'}, 'vdi_tool': {'completed': True, 'completed_at': '2025-07-24T15:32:46Z'}, 'vnc': {'completed': True, 'completed_at': '2025-07-24T15:30:30Z'}} - - -TASK [lock_manager : Get current users updated list] *************************** -ok: [localhost] => { - "ansible_facts": { - "current_users_updated": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Get new usernames list] *********************************** -ok: [localhost] => { - "ansible_facts": { - "new_usernames": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Combine users updated lists] ****************************** -ok: [localhost] => { - "ansible_facts": { - "combined_users_updated": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Debug user deduplication] ********************************* -ok: [localhost] => {} - -MSG: - -User deduplication for vdi_tool: - - current_users_updated: ['alice', 'bob'] - - new_usernames: ['alice', 'bob'] - - combined_users_updated: ['alice', 'bob'] - - -TASK [lock_manager : Get current user secrets for list update] ***************** -ok: [localhost] => { - "ansible_facts": { - "current_user_secrets_for_list": { - "last_secret_check": "2025-07-24T15:32:46Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Update users updated list] ******************************** -ok: [localhost] => { - "ansible_facts": { - "updated_user_secrets_with_list": { - "last_secret_check": "2025-07-24T15:32:46Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - }, - "changed": false -} - -TASK [lock_manager : Apply users updated list] ********************************* -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": true, - "completed_at": "2025-07-24T15:32:46Z" - }, - "vnc": { - "completed": true, - "completed_at": "2025-07-24T15:30:30Z" - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:32:46Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:32:46Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Get current user management data] ************************* -ok: [localhost] => { - "ansible_facts": { - "current_user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - } - }, - "changed": false -} - -TASK [lock_manager : Get current usernames list] ******************************* -ok: [localhost] => { - "ansible_facts": { - "current_usernames": [ - "alice", - "bob" - ] - }, - "changed": false -} - -TASK [lock_manager : Update user management data] ****************************** -ok: [localhost] => { - "ansible_facts": { - "updated_user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - } - }, - "changed": false -} - -TASK [lock_manager : Apply user management update] ***************************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": true, - "completed_at": "2025-07-24T15:32:46Z" - }, - "vnc": { - "completed": true, - "completed_at": "2025-07-24T15:30:30Z" - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:32:46Z", - "lock_version": "1.0", - "setup_status": "configuring", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:32:46Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Debug user management] ************************************ -ok: [localhost] => {} - -MSG: - -User management for vdi_tool: - - current_users: ['alice', 'bob'] - - previous_users: ['alice', 'bob'] - - removed_users: [] - - -TASK [lock_manager : Check if all roles are completed] ************************* -ok: [localhost] => { - "ansible_facts": { - "all_roles_completed": true - }, - "changed": false -} - -TASK [lock_manager : Update setup status based on completion] ****************** -ok: [localhost] => { - "ansible_facts": { - "current_lock_data": { - "completed_roles": { - "base_os": { - "completed": true, - "completed_at": "2025-07-24T15:29:50Z" - }, - "lock_manager": { - "completed": true, - "completed_at": "2025-07-24T15:27:44Z" - }, - "secret_manager": { - "completed": true, - "completed_at": "2025-07-24T15:30:03Z" - }, - "user_provision": { - "completed": true, - "completed_at": "2025-07-24T15:30:16Z" - }, - "vdi_tool": { - "completed": true, - "completed_at": "2025-07-24T15:32:46Z" - }, - "vnc": { - "completed": true, - "completed_at": "2025-07-24T15:30:30Z" - } - }, - "created_at": "2025-07-24T15:27:41Z", - "deployment_hash": "4d7bbe69940a9da1aae5b3acad02ad140e47a546d53111fb6e9b65e460d11aab", - "deployment_name": "vdi-test-scott", - "force_rerun": false, - "last_updated": "2025-07-24T15:32:46Z", - "lock_version": "1.0", - "setup_status": "available", - "user_management": { - "current_users": [ - "alice", - "bob" - ], - "previous_users": [ - "alice", - "bob" - ], - "removed_users": [] - }, - "user_secrets_status": { - "last_secret_check": "2025-07-24T15:32:46Z", - "user_secrets_hash": "6a6a02d3d09933ca282ae2b02336343c9d73701e45bd81475fbd92dcf9d2f3b6", - "users_updated": [ - "alice", - "bob" - ] - } - } - }, - "changed": false -} - -TASK [lock_manager : Debug setup status update] ******************************** -ok: [localhost] => {} - -MSG: - -Setup status update: - - all_roles_completed: True - - setup_status: available - - completed_roles_count: 6 - - total_roles_count: 6 - - -TASK [lock_manager : Write updated lock file] ********************************** -changed: [localhost] => { - "changed": true, - "checksum": "fc55f0f0719e0f714591e3a785b23088e4370d6b", - "dest": "/opt/vdi-setup/.vdi-lock.yaml", - "gid": 0, - "group": "root", - "md5sum": "023467ccf639a5ab5102d3857dac0237", - "mode": "0644", - "owner": "root", - "size": 1106, - "src": "/root/.ansible/tmp/ansible-tmp-1753371167.8426328-29425-109328817609540/source", - "state": "file", - "uid": 0 -} - -TASK [lock_manager : Debug VM metadata update] ********************************* -ok: [localhost] => {} - -MSG: - -VM metadata update info: - - hostname: vdi-test-scott-rocky-0 - - zone: us-central1-a - - lock_file_changed: True - - -TASK [lock_manager : Get lock file content as base64] ************************** -changed: [localhost] => { - "changed": true, - "cmd": "cat /opt/vdi-setup/.vdi-lock.yaml | base64 -w 0", - "delta": "0:00:00.003891", - "end": "2025-07-24 15:32:48.398036", - "rc": 0, - "start": "2025-07-24 15:32:48.394145" -} - -STDOUT: - -dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiB0cnVlCiAgICAgIGNvbXBsZXRlZF9hdDogJzIwMjUtMDctMjRUMTU6MzA6MTZaJwogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjMyOjQ2WicKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiB0cnVlCiAgICAgIGNvbXBsZXRlZF9hdDogJzIwMjUtMDctMjRUMTU6MzA6MzBaJwogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNTozMjo0NlonCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogYXZhaWxhYmxlCiAgdXNlcl9tYW5hZ2VtZW50OgogICAgY3VycmVudF91c2VyczoKICAgIC0gYWxpY2UKICAgIC0gYm9iCiAgICBwcmV2aW91c191c2VyczoKICAgIC0gYWxpY2UKICAgIC0gYm9iCiAgICByZW1vdmVkX3VzZXJzOiBbXQogIHVzZXJfc2VjcmV0c19zdGF0dXM6CiAgICBsYXN0X3NlY3JldF9jaGVjazogJzIwMjUtMDctMjRUMTU6MzI6NDZaJwogICAgdXNlcl9zZWNyZXRzX2hhc2g6IDZhNmEwMmQzZDA5OTMzY2EyODJhZTJiMDIzMzYzNDNjOWQ3MzcwMWU0NWJkODE0NzVmYmQ5MmRjZjlkMmYzYjYKICAgIHVzZXJzX3VwZGF0ZWQ6CiAgICAtIGFsaWNlCiAgICAtIGJvYgo= - -TASK [lock_manager : Add VDI lock file content to VM metadata] ***************** -changed: [localhost] => { - "changed": true, - "cmd": [ - "gcloud", - "compute", - "instances", - "add-metadata", - "vdi-test-scott-rocky-0", - "--metadata", - "vdi-lock-content=dmRpX3NldHVwX3N0YXR1czoKICBjb21wbGV0ZWRfcm9sZXM6CiAgICBiYXNlX29zOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNToyOTo1MFonCiAgICBsb2NrX21hbmFnZXI6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQ0WicKICAgIHNlY3JldF9tYW5hZ2VyOgogICAgICBjb21wbGV0ZWQ6IHRydWUKICAgICAgY29tcGxldGVkX2F0OiAnMjAyNS0wNy0yNFQxNTozMDowM1onCiAgICB1c2VyX3Byb3Zpc2lvbjoKICAgICAgY29tcGxldGVkOiB0cnVlCiAgICAgIGNvbXBsZXRlZF9hdDogJzIwMjUtMDctMjRUMTU6MzA6MTZaJwogICAgdmRpX3Rvb2w6CiAgICAgIGNvbXBsZXRlZDogdHJ1ZQogICAgICBjb21wbGV0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjMyOjQ2WicKICAgIHZuYzoKICAgICAgY29tcGxldGVkOiB0cnVlCiAgICAgIGNvbXBsZXRlZF9hdDogJzIwMjUtMDctMjRUMTU6MzA6MzBaJwogIGNyZWF0ZWRfYXQ6ICcyMDI1LTA3LTI0VDE1OjI3OjQxWicKICBkZXBsb3ltZW50X2hhc2g6IDRkN2JiZTY5OTQwYTlkYTFhYWU1YjNhY2FkMDJhZDE0MGU0N2E1NDZkNTMxMTFmYjZlOWI2NWU0NjBkMTFhYWIKICBkZXBsb3ltZW50X25hbWU6IHZkaS10ZXN0LXNjb3R0CiAgZm9yY2VfcmVydW46IGZhbHNlCiAgbGFzdF91cGRhdGVkOiAnMjAyNS0wNy0yNFQxNTozMjo0NlonCiAgbG9ja192ZXJzaW9uOiAnMS4wJwogIHNldHVwX3N0YXR1czogYXZhaWxhYmxlCiAgdXNlcl9tYW5hZ2VtZW50OgogICAgY3VycmVudF91c2VyczoKICAgIC0gYWxpY2UKICAgIC0gYm9iCiAgICBwcmV2aW91c191c2VyczoKICAgIC0gYWxpY2UKICAgIC0gYm9iCiAgICByZW1vdmVkX3VzZXJzOiBbXQogIHVzZXJfc2VjcmV0c19zdGF0dXM6CiAgICBsYXN0X3NlY3JldF9jaGVjazogJzIwMjUtMDctMjRUMTU6MzI6NDZaJwogICAgdXNlcl9zZWNyZXRzX2hhc2g6IDZhNmEwMmQzZDA5OTMzY2EyODJhZTJiMDIzMzYzNDNjOWQ3MzcwMWU0NWJkODE0NzVmYmQ5MmRjZjlkMmYzYjYKICAgIHVzZXJzX3VwZGF0ZWQ6CiAgICAtIGFsaWNlCiAgICAtIGJvYgo=", - "--zone=us-central1-a" - ], - "delta": "0:00:04.026047", - "end": "2025-07-24 15:32:52.625449", - "failed_when_result": false, - "rc": 0, - "start": "2025-07-24 15:32:48.599402" -} - -STDERR: - -Updated [https://www.googleapis.com/compute/v1/projects/hpc-discovery-external/zones/us-central1-a/instances/vdi-test-scott-rocky-0]. - -TASK [lock_manager : Debug lock file write] ************************************ -ok: [localhost] => {} - -MSG: - -Lock file written: True - -TASK [lock_manager : Verify lock file integrity] ******************************* -ok: [localhost] => { - "changed": false, - "cmd": [ - "python3", - "-c", - "import yaml; yaml.safe_load(open('/opt/vdi-setup/.vdi-lock.yaml'))" - ], - "delta": "0:00:00.040826", - "end": "2025-07-24 15:32:52.906476", - "rc": 0, - "start": "2025-07-24 15:32:52.865650" -} - -TASK [lock_manager : Debug lock file validation] ******************************* -ok: [localhost] => {} - -MSG: - -Lock file validation: True - -TASK [lock_manager : Check if VDI monitor service exists] ********************** -ok: [localhost] => { - "changed": false, - "stat": { - "atime": 1753370874.1548011, - "attr_flags": "", - "attributes": [], - "block_size": 4096, - "blocks": 8, - "charset": "us-ascii", - "checksum": "f6de22707f8d9248014f679a2f1930451e2bf635", - "ctime": 1753370872.6256874, - "dev": 2050, - "device_type": 0, - "executable": false, - "exists": true, - "gid": 0, - "gr_name": "root", - "inode": 285214140, - "isblk": false, - "ischr": false, - "isdir": false, - "isfifo": false, - "isgid": false, - "islnk": false, - "isreg": true, - "issock": false, - "isuid": false, - "mimetype": "text/plain", - "mode": "0644", - "mtime": 1753370872.4766762, - "nlink": 1, - "path": "/etc/systemd/system/vdi-monitor.service", - "pw_name": "root", - "readable": true, - "rgrp": true, - "roth": true, - "rusr": true, - "size": 283, - "uid": 0, - "version": "1338188335", - "wgrp": false, - "woth": false, - "writeable": true, - "wusr": true, - "xgrp": false, - "xoth": false, - "xusr": false - } -} - -TASK [lock_manager : Debug VDI monitor setup conditions] *********************** -ok: [localhost] => {} - -MSG: - -VDI monitor setup conditions: - - current_role: vdi_tool - - vdi_monitor_service_exists: True - - lock_file_creation.changed: False - - will_setup_monitor: False - - -TASK [lock_manager : Setup VDI monitoring service (first time only)] *********** -skipping: [localhost] => { - "changed": false, - "false_condition": "not vdi_monitor_service_stat.stat.exists", - "skip_reason": "Conditional result was False" -} - -TASK [lock_manager : Fail if lock file is invalid] ***************************** -skipping: [localhost] => { - "changed": false, - "false_condition": "lock_file_validation.rc != 0", - "skip_reason": "Conditional result was False" -} - -PLAY RECAP ********************************************************************* -localhost : ok=551 changed=75 unreachable=0 failed=0 skipped=68 rescued=0 ignored=6 - -Thu Jul 24 15:32:53 +0000 2025 Info [1870]: === install.yaml-f44d finished with exit_code=0 === -Thu Jul 24 15:32:53 +0000 2025 Info [1535]: === passed_startup_script.sh-22f5 finished with exit_code=0 === -[root@vdi-test-scott-rocky-0 ~]# \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/main.tf b/community/modules/scripts/vdi-setup/main.tf index 2a338fa919..83d6963eed 100644 --- a/community/modules/scripts/vdi-setup/main.tf +++ b/community/modules/scripts/vdi-setup/main.tf @@ -31,18 +31,20 @@ resource "random_id" "resource_name_suffix" { # Generate vars content using templatefile locals { vdi_vars_content = templatefile("${path.module}/templates/vars.yaml.tftpl", { - deployment_name = var.deployment_name - project_id = var.project_id - user_provision = var.user_provision - vnc_flavor = var.vnc_flavor - vdi_tool = var.vdi_tool - vdi_user_group = var.vdi_user_group - vdi_webapp_port = var.vdi_webapp_port - vdi_resolution = var.vdi_resolution - vdi_users = var.vdi_users - debug = var.debug - vdi_bucket_name = local.bucket_name - zone = var.zone + deployment_name = var.deployment_name + project_id = var.project_id + user_provision = var.user_provision + vnc_flavor = var.vnc_flavor + vdi_tool = var.vdi_tool + vdi_user_group = var.vdi_user_group + vdi_webapp_port = var.vdi_webapp_port + vdi_resolution = var.vdi_resolution + vdi_users = var.vdi_users + debug = var.debug + reset_webapp_admin_password = var.reset_webapp_admin_password + force_rerun = var.force_rerun + vdi_bucket_name = local.bucket_name + zone = var.zone }) } @@ -60,44 +62,55 @@ locals { ansible-galaxy collection install google.cloud EOT }, - # Stage roles.tar.gz + # Stage roles.tar.gz to temporary location { type = "data" source = data.archive_file.roles_tar.output_path destination = "/tmp/vdi/roles.tar.gz" }, - # Unpack into /tmp/vdi/roles + # Unpack into /opt/vdi-setup/roles (final location) { type = "shell" destination = "unpack_roles.sh" content = <<-EOT #!/bin/bash set -eux - mkdir -p /tmp/vdi/roles - tar xzf /tmp/vdi/roles.tar.gz -C /tmp/vdi/roles + mkdir -p /opt/vdi-setup/roles + tar xzf /tmp/vdi/roles.tar.gz -C /opt/vdi-setup/roles + # Clean up temporary file + rm -f /tmp/vdi/roles.tar.gz EOT }, - # write out vars file as YAML + # write out vars file as YAML to final location { type = "data" content = local.vdi_vars_content - destination = "/tmp/vdi/vars.yaml" + destination = "/opt/vdi-setup/vars.yaml" }, - # Run the rendered playbook via ansible-local + # Run the rendered playbook via ansible-local from final location { type = "ansible-local" content = templatefile("${path.module}/templates/install.yaml.tftpl", { - roles = ["lock_manager", "base_os", "secret_manager", "user_provision", "vnc", "vdi_tool"], + roles = ["lock_manager", "base_os", "secret_manager", "user_provision", "vnc", "vdi_tool", "vdi_monitor"], } ) - destination = "/tmp/vdi/install.yaml" - args = var.debug ? "--extra-vars @/tmp/vdi/vars.yaml -v --extra-vars debug=true" : "--extra-vars @/tmp/vdi/vars.yaml" + destination = "/opt/vdi-setup/install.yaml" + args = var.debug ? "--extra-vars @/opt/vdi-setup/vars.yaml -v --extra-vars debug=true" : "--extra-vars @/opt/vdi-setup/vars.yaml" + }, + # Clean up temporary directory + { + type = "shell" + destination = "cleanup.sh" + content = <<-EOT + #!/bin/bash + set -eux + rm -rf /tmp/vdi + EOT }, - # Todo: another runner here to delete /tmp/vdi afterwards? ] bucket_name = "${substr(var.deployment_name, 0, 39)}-vdi-scripts-${random_id.resource_name_suffix.hex}" diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/create_lock.yaml b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/create_lock.yaml index c464599e32..1d1bb1f61b 100644 --- a/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/create_lock.yaml +++ b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/create_lock.yaml @@ -40,6 +40,11 @@ msg: "Lock file created: {{ lock_file_creation.changed }}" when: debug | default(false) and not lock_file_stat.stat.exists +- name: Set fresh deployment flag (only when lock file is created) + ansible.builtin.set_fact: + is_fresh_deployment: true + when: lock_file_creation.changed + - name: Load existing lock file data ansible.builtin.include_vars: file: /opt/vdi-setup/.vdi-lock.yaml @@ -252,7 +257,7 @@ register: lock_file_base64 when: lock_file_write.changed -- name: Add VDI lock file content to VM metadata +- name: Add VDI lock file content to VM metadata (with retry) ansible.builtin.command: > gcloud compute instances add-metadata {{ ansible_hostname }} --metadata vdi-lock-content="{{ lock_file_base64.stdout }}" @@ -261,6 +266,9 @@ - lock_file_write.changed - lock_file_base64.rc == 0 register: metadata_update + retries: 3 + delay: 2 + until: metadata_update.rc == 0 changed_when: metadata_update.rc == 0 failed_when: - metadata_update.rc != 0 @@ -281,28 +289,13 @@ msg: "Lock file validation: {{ lock_file_validation.rc == 0 }}" when: debug | default(false) -- name: Check if VDI monitor service exists - ansible.builtin.stat: - path: /etc/systemd/system/vdi-monitor.service - register: vdi_monitor_service_stat - -- name: Debug VDI monitor setup conditions +- name: Debug lock manager completion ansible.builtin.debug: - msg: | - VDI monitor setup conditions: - - current_role: {{ current_role }} - - vdi_monitor_service_exists: {{ vdi_monitor_service_stat.stat.exists }} - - lock_file_creation.changed: {{ lock_file_creation.changed | default(false) }} - - will_setup_monitor: {{ (not vdi_monitor_service_stat.stat.exists) and (lock_file_creation.changed | default(false)) }} + msg: "Lock manager role completed successfully" when: debug | default(false) -- name: Setup VDI monitoring service (first time only) - ansible.builtin.include_tasks: monitor_setup.yaml - when: - - not vdi_monitor_service_stat.stat.exists - - lock_file_creation.changed - - name: Fail if lock file is invalid ansible.builtin.fail: msg: "Lock file validation failed: {{ lock_file_validation.stderr }}" when: lock_file_validation.rc != 0 + \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/main.yaml index 1f4a1fa6d5..3d236a94d0 100644 --- a/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/main.yaml @@ -58,3 +58,4 @@ when: - current_role == "lock_manager" - lock_operation | default('check') == 'check' + \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/monitor_setup.yaml b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/monitor_setup.yaml deleted file mode 100644 index cacb9573b3..0000000000 --- a/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/monitor_setup.yaml +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -- name: Create VDI monitor script directory - ansible.builtin.file: - path: /usr/local/bin - state: directory - mode: '0755' - -- name: Copy VDI monitor script - ansible.builtin.template: - src: vdi-monitor.sh.j2 - dest: /usr/local/bin/vdi-monitor.sh - mode: '0755' - owner: root - group: root - -- name: Copy VDI monitor systemd service - ansible.builtin.template: - src: vdi-monitor.service.j2 - dest: /etc/systemd/system/vdi-monitor.service - mode: '0644' - owner: root - group: root - -- name: Create VDI monitor log directory - ansible.builtin.file: - path: /var/log - state: directory - mode: '0755' - -- name: Create VDI monitor log files - ansible.builtin.file: - path: "{{ item }}" - state: touch - mode: '0644' - owner: root - group: root - loop: - - /var/log/vdi-monitor.log - - /var/log/ansible-vdi-reconfig.log - -- name: Reload systemd daemon - ansible.builtin.systemd: - daemon_reload: yes - -- name: Enable VDI monitor service - ansible.builtin.systemd: - name: vdi-monitor - enabled: yes - state: started - -- name: Debug VDI monitor setup - ansible.builtin.debug: - msg: | - VDI monitor service setup complete: - - Service: vdi-monitor.service - - Script: /usr/local/bin/vdi-monitor.sh - - Logs: /var/log/vdi-monitor.log, /var/log/ansible-vdi-reconfig.log - - Service enabled and started - when: debug | default(false) diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 b/community/modules/scripts/vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 index 91f0253cf2..b168ccc866 100644 --- a/community/modules/scripts/vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 +++ b/community/modules/scripts/vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 @@ -11,10 +11,10 @@ vdi_setup_status: setup_status: "configuring" # configuring, available, error completed_roles: - base_os: - completed: false lock_manager: completed: false + base_os: + completed: false secret_manager: completed: false user_provision: @@ -23,6 +23,8 @@ vdi_setup_status: completed: false vdi_tool: completed: false + vdi_monitor: + completed: false user_secrets_status: last_secret_check: "{{ ansible_date_time.iso8601 }}" @@ -33,3 +35,4 @@ vdi_setup_status: current_users: [] previous_users: [] removed_users: [] + \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/templates/vdi-monitor.sh.j2 b/community/modules/scripts/vdi-setup/roles/lock_manager/templates/vdi-monitor.sh.j2 deleted file mode 100644 index 6a8bf54614..0000000000 --- a/community/modules/scripts/vdi-setup/roles/lock_manager/templates/vdi-monitor.sh.j2 +++ /dev/null @@ -1,184 +0,0 @@ -#!/bin/bash - -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# VDI Configuration Monitor -# Monitors the VDI lock file for changes and triggers re-configuration - -LOCK_FILE="/opt/vdi-setup/.vdi-lock.yaml" -CONFIG_HASH_FILE="/opt/vdi-setup/.config-hash" -MONITOR_LOG="/var/log/vdi-monitor.log" -ANSIBLE_LOG="/var/log/ansible-vdi-reconfig.log" -BUCKET_NAME="{{ vdi_bucket_name }}" -VDI_SETUP_DIR="/opt/vdi-setup" - -# Function to log messages -log_message() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$MONITOR_LOG" -} - -# Function to sync files from bucket -sync_from_bucket() { - log_message "Syncing files from bucket: gs://$BUCKET_NAME" - - # Create temporary directory for sync - local temp_dir=$(mktemp -d) - - # Download files from bucket - if gsutil -m rsync -r "gs://$BUCKET_NAME/" "$temp_dir/"; then - log_message "Successfully downloaded files from bucket" - - # Copy files to VDI setup directory - if [[ -d "$temp_dir/roles" ]]; then - cp -r "$temp_dir/roles" "$VDI_SETUP_DIR/" - log_message "Updated roles directory" - fi - - if [[ -f "$temp_dir/install.yaml" ]]; then - cp "$temp_dir/install.yaml" "$VDI_SETUP_DIR/" - log_message "Updated install.yaml" - fi - - if [[ -f "$temp_dir/vars.yaml" ]]; then - cp "$temp_dir/vars.yaml" "$VDI_SETUP_DIR/" - log_message "Updated vars.yaml" - fi - - # Clean up temporary directory - rm -rf "$temp_dir" - return 0 - else - log_message "ERROR: Failed to sync files from bucket" - rm -rf "$temp_dir" - return 1 - fi -} - -# Function to get current configuration hash -get_config_hash() { - if [[ -f "$LOCK_FILE" ]]; then - # Extract deployment_hash and user_secrets_hash from lock file - deployment_hash=$(grep "deployment_hash:" "$LOCK_FILE" | awk '{print $2}' | tr -d '"') - user_secrets_hash=$(grep "user_secrets_hash:" "$LOCK_FILE" | awk '{print $2}' | tr -d '"') - echo "${deployment_hash}-${user_secrets_hash}" - else - echo "no-lock-file" - fi -} - -# Function to check if configuration has changed -check_config_changed() { - local current_hash=$(get_config_hash) - local stored_hash="" - - if [[ -f "$CONFIG_HASH_FILE" ]]; then - stored_hash=$(cat "$CONFIG_HASH_FILE") - fi - - if [[ "$current_hash" != "$stored_hash" ]]; then - log_message "Configuration change detected: $stored_hash -> $current_hash" - return 0 - else - return 1 - fi -} - -# Function to update stored hash -update_stored_hash() { - local current_hash=$(get_config_hash) - echo "$current_hash" > "$CONFIG_HASH_FILE" - log_message "Updated stored hash: $current_hash" -} - -# Function to trigger VDI re-configuration -trigger_reconfig() { - log_message "Triggering VDI re-configuration" - - # First, sync latest files from bucket - if ! sync_from_bucket; then - log_message "WARNING: Failed to sync from bucket, proceeding with local files" - fi - - # Change to VDI setup directory - cd "$VDI_SETUP_DIR" || { - log_message "ERROR: Cannot change to $VDI_SETUP_DIR" - return 1 - } - - # Run Ansible playbook with re-configuration - ansible-playbook install.yaml \ - --connection=local \ - --inventory=localhost, \ - --limit=localhost \ - --extra-vars="force_rerun=true" \ - --extra-vars="debug=true" \ - >> "$ANSIBLE_LOG" 2>&1 - - local exit_code=$? - if [[ $exit_code -eq 0 ]]; then - log_message "VDI re-configuration completed successfully" - update_stored_hash - else - log_message "ERROR: VDI re-configuration failed with exit code $exit_code" - log_message "Check $ANSIBLE_LOG for details" - fi - - return $exit_code -} - -# Main monitoring loop -main() { - log_message "VDI Configuration Monitor started" - - # Initialize stored hash if not exists - if [[ ! -f "$CONFIG_HASH_FILE" ]]; then - update_stored_hash - fi - - # Main monitoring loop - while true; do - if check_config_changed; then - trigger_reconfig - fi - - # Sleep for 60 seconds before next check - sleep 60 - done -} - -# Test mode function -test_mode() { - log_message "VDI Configuration Monitor - Test Mode" - log_message "Testing bucket sync functionality..." - - if sync_from_bucket; then - log_message "SUCCESS: Bucket sync test passed" - exit 0 - else - log_message "FAILED: Bucket sync test failed" - exit 1 - fi -} - -# Handle script termination -trap 'log_message "VDI Configuration Monitor stopped"; exit 0' SIGTERM SIGINT - -# Check for test mode -if [[ "${1:-}" == "--test" ]]; then - test_mode -fi - -# Start monitoring -main diff --git a/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml index e7b5f69ce0..6bbd23b9b3 100644 --- a/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml @@ -37,94 +37,131 @@ # Run secret manager tasks only if needed - name: Run secret manager tasks block: - # General secret handling - - name: Generate random database password - ansible.builtin.set_fact: - database_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}" - - - name: Generate random webapp admin password - ansible.builtin.set_fact: - webapp_admin_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}" - - - name: Compute Webapp admin password hash to save to Webapp server - ansible.builtin.set_fact: - webapp_admin_hash: "{{ webapp_admin_password | hash('sha256') | upper }}" - - - name: Get access token for Secret Manager API - ansible.builtin.uri: - url: "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" - method: GET - headers: - Metadata-Flavor: Google - return_content: true - register: access_token_result - - - name: Create a GCP secret for webapp server login - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets?secretId=webapp-server-password-{{ deployment_name }}" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - replication: - automatic: {} - status_code: [200, 409] # 409 = already exists - register: webapp_secret_create - - - name: Add webapp password version to secret - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/webapp-server-password-{{ deployment_name }}:addVersion" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - payload: - data: "{{ webapp_admin_password | b64encode }}" - register: webapp_secret - - - name: Create a GCP secret for database password - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets?secretId=db-password-{{ deployment_name }}" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - replication: - automatic: {} - status_code: [200, 409] # 409 = already exists - register: database_secret_create - - - name: Add database password version to secret - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/db-password-{{ deployment_name }}:addVersion" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - payload: - data: "{{ database_password | b64encode }}" - register: database_secret - - # Per-user secret handling - - name: Generate per-user secrets and enrich list - include_tasks: user_secret_tasks.yaml - loop: "{{ vdi_users }}" - loop_control: - loop_var: item - - - name: Set enriched vdi_users_updated variable - ansible.builtin.set_fact: - vdi_users_updated: "{{ vdi_users_updated }}" - when: vdi_users_updated is defined - + # General secret handling + - name: Generate random database password + ansible.builtin.set_fact: + database_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}" + + - name: Get access token for Secret Manager API + ansible.builtin.uri: + url: "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" + method: GET + headers: + Metadata-Flavor: Google + return_content: true + register: access_token_result + + # Webapp admin password handling + - name: Fetch existing webapp admin password from Secret Manager + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/webapp-server-password-{{ deployment_name }}/versions/latest:access" + method: GET + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + return_content: true + register: webapp_admin_sm_result + failed_when: false + + - name: Set webapp admin password from Secret Manager (if available and not resetting) + ansible.builtin.set_fact: + webapp_admin_password: "{{ webapp_admin_sm_result.json.payload.data | b64decode }}" + webapp_password_was_fetched: true + when: + - webapp_admin_sm_result.status == 200 + - not (reset_webapp_admin_password | default(false)) + + - name: Generate new webapp admin password (if not available from Secret Manager or reset requested) + ansible.builtin.set_fact: + webapp_admin_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}" + webapp_password_was_fetched: false + when: + - webapp_admin_sm_result.status != 200 or (reset_webapp_admin_password | default(false)) + + - name: Compute Webapp admin password hash to save to Webapp server + ansible.builtin.set_fact: + webapp_admin_hash: "{{ webapp_admin_password | hash('sha256') | upper }}" + + - name: Create a GCP secret for webapp server login + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets?secretId=webapp-server-password-{{ deployment_name }}" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + replication: + automatic: {} + status_code: [200, 409] # 409 = already exists + register: webapp_secret_create + + - name: Add webapp password version to secret (only if new password was generated) + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/webapp-server-password-{{ deployment_name }}:addVersion" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + payload: + data: "{{ webapp_admin_password | b64encode }}" + register: webapp_secret + when: not (webapp_password_was_fetched | default(false)) + + - name: Debug webapp admin password handling + ansible.builtin.debug: + msg: | + Webapp admin password handling (secret_manager): + - reset_requested: {{ reset_webapp_admin_password | default(false) }} + - is_fresh_deployment: {{ is_fresh_deployment | default(false) }} + - reset_applied: {{ reset_webapp_admin_password | default(false) }} + - secret_manager_status: {{ webapp_admin_sm_result.status }} + - password_source: {{ 'Secret Manager' if webapp_admin_sm_result.status == 200 and not (reset_webapp_admin_password | default(false)) else 'Generated' }} + - password_was_fetched: {{ webapp_password_was_fetched | default(false) }} + - password_length: {{ webapp_admin_password | length if webapp_admin_password is defined else 'N/A' }} + - stored_in_secret_manager: {{ not (webapp_password_was_fetched | default(false)) }} + when: debug | default(false) + + - name: Create a GCP secret for database password + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets?secretId=db-password-{{ deployment_name }}" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + replication: + automatic: {} + status_code: [200, 409] # 409 = already exists + register: database_secret_create + + - name: Add database password version to secret + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/db-password-{{ deployment_name }}:addVersion" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + payload: + data: "{{ database_password | b64encode }}" + register: database_secret + + # Per-user secret handling + - name: Generate per-user secrets and enrich list + include_tasks: user_secret_tasks.yaml + loop: "{{ vdi_users }}" + loop_control: + loop_var: item + + - name: Set enriched vdi_users_updated variable + ansible.builtin.set_fact: + vdi_users_updated: "{{ vdi_users_updated }}" + when: vdi_users_updated is defined when: role_should_run # Mark role as completed diff --git a/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml b/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml index aafe3954bf..a52d694b34 100644 --- a/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml +++ b/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml @@ -67,11 +67,34 @@ vdiuser_password: "{{ sm_result.json.payload.data | b64decode }}" when: password_source == "secret" and sm_result.status == 200 +- name: Check if password should be forced to update (only on existing deployments) + ansible.builtin.set_fact: + password_force_update: "{{ item.reset_password | default(false) and not (is_fresh_deployment | default(false)) }}" + - name: Check if password has actually changed for {{ item.username }} ansible.builtin.set_fact: - password_actually_changed: "{{ user_config_changed or (password_source == 'secret' and sm_result.status == 200 and vdiuser_password != item.password | default('')) }}" + password_actually_changed: "{{ user_config_changed or password_force_update }}" when: password_source == "secret" and sm_result.status == 200 +- name: Handle failed Secret Manager fetch with reset_password (only on existing deployments) + ansible.builtin.debug: + msg: | + WARNING: Failed to fetch password from Secret Manager for {{ item.username }} + but reset_password is set. This will trigger password regeneration. + Secret: {{ item.secret_name }} + Status: {{ sm_result.status }} + when: password_source == "secret" and sm_result.status != 200 and item.reset_password | default(false) and not (is_fresh_deployment | default(false)) + +- name: Set password_actually_changed for failed Secret Manager fetch with reset_password + ansible.builtin.set_fact: + password_actually_changed: true + when: password_source == "secret" and sm_result.status != 200 and item.reset_password | default(false) and not (is_fresh_deployment | default(false)) + +- name: Generate new password for user with reset_password when Secret Manager fetch fails + ansible.builtin.set_fact: + vdiuser_password: "{{ lookup('password', '/dev/null length=16 chars=ascii_letters,digits') }}" + when: password_source == "secret" and sm_result.status != 200 and item.reset_password | default(false) and not (is_fresh_deployment | default(false)) + - name: Debug password change detection for {{ item.username }} ansible.builtin.debug: msg: | @@ -79,7 +102,11 @@ - user_config_changed: {{ user_config_changed }} - password_source: {{ password_source }} - sm_result_status: {{ sm_result.status }} + - reset_password: {{ item.reset_password | default(false) }} + - is_fresh_deployment: {{ is_fresh_deployment | default(false) }} + - password_force_update: {{ password_force_update }} - password_actually_changed: {{ password_actually_changed | default(false) }} + - vdiuser_password_set: {{ vdiuser_password is defined }} when: debug | default(false) and password_source == "secret" - name: Generate random password for user {{ item.username }} @@ -92,6 +119,24 @@ vdiuser_password: "{{ item.password }}" when: password_source == "password" +- name: Set password_actually_changed for generate and password sources + ansible.builtin.set_fact: + password_actually_changed: "{{ user_config_changed or (item.reset_password | default(false) and not (is_fresh_deployment | default(false))) }}" + when: password_source in ["generate", "password"] + +- name: Debug final password decision for {{ item.username }} + ansible.builtin.debug: + msg: | + Final password decision for {{ item.username }}: + - password_source: {{ password_source }} + - user_config_changed: {{ user_config_changed }} + - reset_password: {{ item.reset_password | default(false) }} + - is_fresh_deployment: {{ is_fresh_deployment | default(false) }} + - password_actually_changed: {{ password_actually_changed | default(false) }} + - vdiuser_password_set: {{ vdiuser_password is defined }} + - password_length: {{ vdiuser_password | length if vdiuser_password is defined else 'N/A' }} + when: debug | default(false) + - name: Create secret for user {{ item.username }} (provided password) ansible.builtin.uri: url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets?secretId=vdi-user-password-{{ item.username }}-{{ deployment_name }}" @@ -146,6 +191,19 @@ data: "{{ vdiuser_password | b64encode }}" when: password_source == "generate" +- name: Update Secret Manager for user with reset_password when Secret Manager fetch failed + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets/{{ item.secret_name }}:addVersion" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + payload: + data: "{{ vdiuser_password | b64encode }}" + when: password_source == "secret" and sm_result.status != 200 and item.reset_password | default(false) + - name: Set VNC configuration for user ansible.builtin.set_fact: vnc_config: >- diff --git a/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml index bbeb26975c..dc841d507b 100644 --- a/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml @@ -37,6 +37,7 @@ # Run user provision tasks only if needed - name: Run user provision tasks block: + - name: Ensure 'wheel' group is present group: name: wheel diff --git a/community/modules/scripts/vdi-setup/roles/vdi_monitor/README.md b/community/modules/scripts/vdi-setup/roles/vdi_monitor/README.md new file mode 100644 index 0000000000..d94bf42b9e --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vdi_monitor/README.md @@ -0,0 +1,200 @@ +# VDI Monitor Role + +## Overview + +The `vdi_monitor` role sets up and configures the VDI Configuration Monitor system. This role is responsible for: + +- Installing the modular VDI monitoring scripts +- Configuring the systemd service for automatic monitoring +- Setting up file change detection +- Enabling automatic VDI reconfiguration + +## Features + +### šŸ” Change Detection +- **File Monitoring**: Detects changes to `vars.yaml`, `install.yaml`, and `roles.tar.gz` in GCS bucket +- **Password Reset Detection**: Detects `reset_password: true` flags in configuration +- **Immediate Initialization**: Refreshes file tracking on startup to capture current bucket state + +### āš” Automatic Reconfiguration +- **Immediate Action**: Triggers reconfiguration as soon as changes are detected +- **Cooldown**: Prevents excessive reconfigurations with a 1-minute cooldown period +- **Idempotent**: Safe to run multiple times without side effects +- **Smart Container Management**: User changes handled via SQL updates, full container recreation only when necessary + +### šŸ› ļø Modular Architecture +- **Config Module**: Constants, logging, and initialization +- **File Monitor**: GCS file detection and synchronization +- **Change Detector**: Change detection logic and reconfiguration +- **Test Utils**: Comprehensive testing and diagnostics + +## Role Structure + +``` +vdi_monitor/ +ā”œā”€ā”€ defaults/ +ā”‚ ā””ā”€ā”€ main.yaml # Default variables +ā”œā”€ā”€ tasks/ +ā”‚ ā””ā”€ā”€ main.yaml # Main role tasks +ā”œā”€ā”€ templates/ +ā”‚ ā”œā”€ā”€ vdi-monitor.sh.j2 # Main orchestrator script +ā”‚ ā”œā”€ā”€ vdi-monitor-config.sh.j2 # Configuration module +ā”‚ ā”œā”€ā”€ vdi-monitor-file.sh.j2 # File monitoring module +ā”‚ ā”œā”€ā”€ vdi-monitor-detector.sh.j2 # Change detection module +ā”‚ ā”œā”€ā”€ vdi-monitor-test.sh.j2 # Testing module +ā”‚ ā””ā”€ā”€ vdi-monitor.service.j2 # Systemd service +ā”œā”€ā”€ meta/ +ā”‚ ā””ā”€ā”€ main.yaml # Role metadata +ā””ā”€ā”€ README.md # This file +``` + +## Variables + +### Required Variables +None - all variables have sensible defaults. + +### Optional Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `vdi_monitor_check_interval` | `60` | Monitoring check interval in seconds | +| `vdi_monitor_reconfig_cooldown` | `60` | Reconfiguration cooldown in seconds (1 minute) | +| `vdi_monitor_log_file` | `"/var/log/vdi-monitor.log"` | Monitor log file | +| `vdi_monitor_ansible_log_file` | `"/var/log/ansible-vdi-reconfig.log"` | Ansible reconfig log | + +## Usage + +### Basic Usage +The role is automatically included in the VDI setup playbook: + +```yaml +# In your VDI blueprint +- id: vdi_setup + source: modules/scripts/vdi-setup + # ... other configuration +``` + +### Manual Role Execution +```bash +# Run the role manually +ansible-playbook -i localhost, --connection=local \ + -e "role=vdi_monitor" \ + /opt/vdi-setup/install.yaml +``` + +## Monitoring Commands + +### Test Mode +```bash +# Run comprehensive tests +sudo /usr/local/bin/vdi-monitor.sh --test +``` + +### Diagnostics +```bash +# Run system diagnostics +sudo /usr/local/bin/vdi-monitor.sh --diagnostics +``` + +### Help +```bash +# Show help and usage +sudo /usr/local/bin/vdi-monitor.sh --help +``` + +## Service Management + +### Check Service Status +```bash +systemctl status vdi-monitor +``` + +### View Logs +```bash +# Monitor logs +tail -f /var/log/vdi-monitor.log + +# Ansible reconfiguration logs +tail -f /var/log/ansible-vdi-reconfig.log +``` + +### Manual Service Control +```bash +# Start service +sudo systemctl start vdi-monitor + +# Stop service +sudo systemctl stop vdi-monitor + +# Restart service +sudo systemctl restart vdi-monitor + +# Enable/disable service +sudo systemctl enable vdi-monitor +sudo systemctl disable vdi-monitor +``` + +## Dependencies + +- `lock_manager` role (for deployment locking) +- Google Cloud SDK (`gcloud`, `gsutil`) +- Ansible (`ansible-playbook`) +- Systemd (for service management) + +## Files Created + +### Scripts +- `/usr/local/bin/vdi-monitor.sh` - Main monitoring script +- `/opt/vdi-setup/vdi-monitor-lib/` - Module directory + - `config.sh` - Configuration module + - `file-monitor.sh` - File monitoring module + - `change-detector.sh` - Change detection module + - `test-utils.sh` - Testing module + +### Services +- `/etc/systemd/system/vdi-monitor.service` - Systemd service + +### Logs +- `/var/log/vdi-monitor.log` - Monitor activity log +- `/var/log/ansible-vdi-reconfig.log` - Reconfiguration log + +### State Files +- `/opt/vdi-setup/.current_*_file` - File tracking state +- `/opt/vdi-setup/.pending_changes` - Pending change details + +## How It Works + +### Change Detection Process +1. **File Monitoring**: Every 60 seconds, checks for changes in GCS bucket files +2. **Immediate Response**: When changes are detected, triggers reconfiguration immediately +3. **File Sync**: Downloads updated files from GCS bucket +4. **Ansible Execution**: Runs the VDI setup playbook with updated configuration +5. **Cooldown**: Prevents reconfigurations for 1 minute after successful execution + +### Password Reset Workflow +1. **Blueprint Update**: User adds `reset_password: true` to a user in blueprint +2. **Terraform Deployment**: New `vars.yaml` is uploaded to GCS with new file suffix +3. **Change Detection**: VDI monitor detects the new file suffix +4. **Reconfiguration**: Ansible roles re-run with `reset_password: true` flag +5. **Password Update**: New password is generated and stored in Secret Manager +6. **Container Preservation**: User changes are handled via SQL updates without recreating containers + +## Troubleshooting + +### Common Issues + +1. **Service won't start** + - Check logs: `journalctl -u vdi-monitor` + - Verify permissions: `ls -la /usr/local/bin/vdi-monitor.sh` + - Check dependencies: `which gcloud gsutil ansible-playbook` + +2. **No changes detected** + - Run test mode: `sudo /usr/local/bin/vdi-monitor.sh --test` + - Check bucket access: `gsutil ls gs://your-bucket-name/` + - Verify file tracking: Check `.current_*_file` files in `/opt/vdi-setup/` + - Check bucket name retrieval: Look for "DEBUG: Bucket name" in logs + +3. **Ansible not running after change detection** + - Check Ansible logs: `tail -f /var/log/ansible-vdi-reconfig.log` + - Verify files exist: `ls -la /opt/vdi-setup/install.yaml /opt/vdi-setup/vars.yaml` + - Run diagnostics: `sudo /usr/local/bin/vdi-monitor.sh --diagnostics` diff --git a/community/modules/scripts/vdi-setup/roles/vdi_monitor/defaults/main.yaml b/community/modules/scripts/vdi-setup/roles/vdi_monitor/defaults/main.yaml new file mode 100644 index 0000000000..7cf155be2e --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vdi_monitor/defaults/main.yaml @@ -0,0 +1,25 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# VDI Monitor Role Defaults + +# Monitoring configuration +vdi_monitor_check_interval: 60 # seconds +vdi_monitor_reconfig_cooldown: 60 # seconds (1 minute) + +# Logging +vdi_monitor_log_file: "/var/log/vdi-monitor.log" +vdi_monitor_ansible_log_file: "/var/log/ansible-vdi-reconfig.log" + \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/roles/vdi_monitor/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/vdi_monitor/tasks/main.yaml new file mode 100644 index 0000000000..ed932263bc --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vdi_monitor/tasks/main.yaml @@ -0,0 +1,161 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: DEBUG About to import lock_manager for vdi_monitor + debug: + msg: "About to import lock_manager for vdi_monitor" + +# Check if this role should run +- name: Set current role for lock manager check + ansible.builtin.set_fact: + current_role: "vdi_monitor" + +- name: Check if vdi_monitor role should run + ansible.builtin.import_role: + name: lock_manager + tasks_from: check_lock + register: lock_check_result + +- name: DEBUG Finished import_role for vdi_monitor + debug: + msg: "Finished import_role for vdi_monitor" + +- name: Set role_should_run from lock check result + ansible.builtin.set_fact: + role_should_run: "{{ lock_check_result.ansible_facts.role_should_run | default(true) }}" + +# Skip all tasks if role should not run +- name: Skip vdi_monitor tasks if role should not run + ansible.builtin.debug: + msg: "Skipping vdi_monitor role - already completed or not needed" + when: not role_should_run + +- name: Debug VDI monitor role execution + ansible.builtin.debug: + msg: "VDI monitor role started - operation: {{ monitor_operation | default('setup') }}" + when: role_should_run + +- name: Ensure VDI setup directory exists + ansible.builtin.file: + path: /opt/vdi-setup + state: directory + mode: '0755' + register: vdi_dir_creation + when: role_should_run + +- name: Create VDI monitor module directory + ansible.builtin.file: + path: /opt/vdi-setup/vdi-monitor-lib + state: directory + mode: '0755' + register: monitor_lib_creation + when: role_should_run + +- name: Debug directory creation status + ansible.builtin.debug: + msg: "VDI dir created: {{ vdi_dir_creation.changed }}, Monitor lib created: {{ monitor_lib_creation.changed }}" + when: debug | default(false) and role_should_run + +# Copy VDI monitor modules +- name: Copy VDI monitor configuration module + ansible.builtin.template: + src: vdi-monitor-config.sh.j2 + dest: /opt/vdi-setup/vdi-monitor-lib/config.sh + mode: '0755' + register: config_module_copy + when: role_should_run + +- name: Copy VDI monitor file module + ansible.builtin.template: + src: vdi-monitor-file.sh.j2 + dest: /opt/vdi-setup/vdi-monitor-lib/file-monitor.sh + mode: '0755' + register: file_module_copy + when: role_should_run + +- name: Copy VDI monitor change detector module + ansible.builtin.template: + src: vdi-monitor-detector.sh.j2 + dest: /opt/vdi-setup/vdi-monitor-lib/change-detector.sh + mode: '0755' + register: detector_module_copy + when: role_should_run + +- name: Copy VDI monitor test module + ansible.builtin.template: + src: vdi-monitor-test.sh.j2 + dest: /opt/vdi-setup/vdi-monitor-lib/test-utils.sh + mode: '0755' + register: test_module_copy + when: role_should_run + +- name: Copy VDI monitor main script + ansible.builtin.template: + src: vdi-monitor.sh.j2 + dest: /usr/local/bin/vdi-monitor.sh + mode: '0755' + register: main_script_copy + when: role_should_run + +- name: Copy VDI monitor systemd service + ansible.builtin.template: + src: vdi-monitor.service.j2 + dest: /etc/systemd/system/vdi-monitor.service + mode: '0644' + register: service_copy + when: role_should_run + +- name: Debug module copy status + ansible.builtin.debug: + msg: "VDI monitor modules copied - config: {{ config_module_copy.changed }}, file: {{ file_module_copy.changed }}, detector: {{ detector_module_copy.changed }}, test: {{ test_module_copy.changed }}, main: {{ main_script_copy.changed }}, service: {{ service_copy.changed }}" + when: debug | default(false) and role_should_run + +# Enable and start VDI monitor service +- name: Reload systemd daemon + ansible.builtin.systemd: + daemon_reload: yes + when: service_copy.changed and role_should_run + +- name: Enable VDI monitor service + ansible.builtin.systemd: + name: vdi-monitor + enabled: yes + state: started + register: monitor_service_start + when: role_should_run + +- name: Debug VDI monitor service status + ansible.builtin.debug: + msg: "VDI monitor service started: {{ monitor_service_start.changed }}" + when: debug | default(false) and role_should_run + +- name: Debug VDI monitor role completion + ansible.builtin.debug: + msg: "VDI monitor role completed successfully" + when: role_should_run + +# Mark role as completed +- name: Set current role for lock manager completion + ansible.builtin.set_fact: + current_role: "vdi_monitor" + role_completed: true + when: role_should_run + +- name: Mark vdi_monitor role as completed + ansible.builtin.import_role: + name: lock_manager + tasks_from: create_lock + when: role_should_run + \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-config.sh.j2 b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-config.sh.j2 new file mode 100644 index 0000000000..9df33ef5b7 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-config.sh.j2 @@ -0,0 +1,138 @@ +#!/bin/bash + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# VDI Monitor Configuration Module +# Contains constants, logging functions, and initialization logic + +# Constants +LOCK_FILE="/opt/vdi-setup/.vdi-lock.yaml" +CONFIG_HASH_FILE="/opt/vdi-setup/.config-hash" +MONITOR_LOG="/var/log/vdi-monitor.log" +ANSIBLE_LOG="/var/log/ansible-vdi-reconfig.log" +VDI_SETUP_DIR="/opt/vdi-setup" + +# Reconfiguration cooldown (prevents rapid reconfigurations) +RECONFIG_COOLDOWN={{ vdi_monitor_reconfig_cooldown | default(300) }} # seconds between reconfigurations + +# Function to log messages +log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$MONITOR_LOG" +} + +# Function to log messages without stdout output (for functions that return values) +log_message_quiet() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$MONITOR_LOG" +} + +# Function to get bucket name from vars file +get_bucket_name() { + local vars_file="$VDI_SETUP_DIR/vars.yaml" + + log_message_quiet "DEBUG: Getting bucket name from: $vars_file" + + if [[ -f "$vars_file" ]]; then + log_message_quiet "DEBUG: vars.yaml file exists" + local bucket_name=$(grep "^vdi_bucket_name:" "$vars_file" | awk '{print $2}' | tr -d ' ') + log_message_quiet "DEBUG: Extracted bucket name: '$bucket_name'" + if [[ -n "$bucket_name" ]]; then + echo "$bucket_name" + return 0 + else + log_message_quiet "WARNING: Bucket name is empty after extraction" + fi + else + log_message_quiet "WARNING: vars.yaml file not found: $vars_file" + fi + + echo "" + return 1 +} + +# Function to validate environment +validate_environment() { + # Check if VDI setup directory exists + if [[ ! -d "$VDI_SETUP_DIR" ]]; then + log_message "ERROR: VDI setup directory not found: $VDI_SETUP_DIR" + return 1 + fi + + # Check if vars.yaml exists + if [[ ! -f "$VDI_SETUP_DIR/vars.yaml" ]]; then + log_message "ERROR: vars.yaml not found: $VDI_SETUP_DIR/vars.yaml" + return 1 + fi + + # Get and validate bucket name + local bucket_name=$(get_bucket_name) + if [[ -z "$bucket_name" ]]; then + log_message "ERROR: Could not extract bucket name from vars.yaml" + return 1 + fi + + # Check if gcloud is available + if ! command -v gcloud >/dev/null 2>&1; then + log_message "ERROR: gcloud command not found" + return 1 + fi + + # Check if gsutil is available + if ! command -v gsutil >/dev/null 2>&1; then + log_message "ERROR: gsutil command not found" + return 1 + fi + + log_message "Environment validation passed" + return 0 +} + +# Function to create necessary directories and files +setup_directories() { + # Ensure log directory exists + mkdir -p "$(dirname "$MONITOR_LOG")" + + # Ensure VDI setup directory exists + mkdir -p "$VDI_SETUP_DIR" + + # Create log files if they don't exist + touch "$MONITOR_LOG" + touch "$ANSIBLE_LOG" + + log_message "Directories and files setup complete" +} + +# Function to initialize monitoring system +initialize_monitoring() { + log_message "Initializing VDI monitoring system" + + # Setup directories + setup_directories + + # Validate environment + if ! validate_environment; then + log_message "ERROR: Environment validation failed" + exit 1 + fi + + # Get bucket name + BUCKET_NAME=$(get_bucket_name) + if [[ -z "$BUCKET_NAME" ]]; then + log_message "ERROR: Bucket name not available. Cannot proceed with monitoring." + exit 1 + fi + + log_message "Using bucket: gs://$BUCKET_NAME" + log_message "VDI monitoring system initialized successfully" +} \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-detector.sh.j2 b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-detector.sh.j2 new file mode 100644 index 0000000000..9fbcf364b3 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-detector.sh.j2 @@ -0,0 +1,341 @@ +#!/bin/bash + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# VDI Monitor Change Detection Module +# Contains change detection logic and reconfiguration functions + +# Function to check if monitoring system is initialized +is_system_initialized() { + local vars_file="$VDI_SETUP_DIR/vars.yaml" + if [[ ! -f "$vars_file" ]]; then + return 1 + fi + + # Check if file tracking is initialized for all required files + if [[ ! -f "/opt/vdi-setup/.current_vars_file" ]] || \ + [[ ! -f "/opt/vdi-setup/.current_install_file" ]] || \ + [[ ! -f "/opt/vdi-setup/.current_roles_file" ]]; then + return 1 + fi + + return 0 # Fully initialized +} + +# Function to check if configuration files have changed +check_config_changed() { + local changed=false + local change_details="" + + # Check vars.yaml + local current_vars_file=$(get_current_file_name "vars.yaml") + local stored_vars_file=$(get_stored_file_name "vars") + + if [[ "$current_vars_file" != "$stored_vars_file" ]]; then + change_details+="vars.yaml: $stored_vars_file -> $current_vars_file; " + changed=true + fi + + # Check install.yaml + local current_install_file=$(get_current_file_name "install.yaml") + local stored_install_file=$(get_stored_file_name "install") + + if [[ "$current_install_file" != "$stored_install_file" ]]; then + change_details+="install.yaml: $stored_install_file -> $current_install_file; " + changed=true + fi + + # Check roles.tar.gz + local current_roles_file=$(get_current_file_name "roles.tar.gz") + local stored_roles_file=$(get_stored_file_name "roles") + + if [[ "$current_roles_file" != "$stored_roles_file" ]]; then + change_details+="roles.tar.gz: $stored_roles_file -> $current_roles_file; " + changed=true + fi + + # Check for reset_password flags + if check_reset_password_flags; then + change_details+="reset_password flags detected; " + changed=true + fi + + # Store change details for logging + if [[ "$changed" == "true" ]]; then + echo "$change_details" > "/opt/vdi-setup/.pending_changes" + log_message "Configuration changes detected: $change_details" + return 0 + else + return 1 + fi +} + +# Function to check if any user has reset_password flags set +check_reset_password_flags() { + local vars_file="$VDI_SETUP_DIR/vars.yaml" + if [[ ! -f "$vars_file" ]]; then + return 1 + fi + + local in_users_section=false + local has_reset_flag=false + + while IFS= read -r line; do + # Check if we're in the vdi_users section + if [[ "$line" =~ ^[[:space:]]*vdi_users:[[:space:]]*$ ]]; then + in_users_section=true + continue + fi + + # Check if we've left the vdi_users section + if [[ "$in_users_section" == "true" && "$line" =~ ^[[:space:]]*[a-zA-Z] ]]; then + in_users_section=false + continue + fi + + if [[ "$in_users_section" == "true" ]]; then + # Check for reset_password flag + if [[ "$line" =~ reset_password:[[:space:]]*(true|yes|1) ]]; then + has_reset_flag=true + log_message "DEBUG: Found reset_password flag set to true" + break + fi + fi + done < "$vars_file" + + if [[ "$has_reset_flag" == "true" ]]; then + return 0 + else + return 1 + fi +} + +# Function to trigger VDI re-configuration +trigger_reconfig() { + log_message "Triggering VDI re-configuration" + + # Update deployment status to "reconfiguring" + update_deployment_status "reconfiguring" + + # Log pending changes if available + if [[ -f "/opt/vdi-setup/.pending_changes" ]]; then + local pending_changes=$(cat "/opt/vdi-setup/.pending_changes") + log_message "Pending changes to apply: $pending_changes" + fi + + # First, sync latest files from bucket + if ! sync_changed_files; then + log_message "WARNING: No files changed or failed to sync, proceeding with local files" + fi + + # Change to VDI setup directory + cd "$VDI_SETUP_DIR" || { + log_message "ERROR: Cannot change to $VDI_SETUP_DIR" + update_deployment_status "available" + return 1 + } + + # Verify files exist before running Ansible + log_message "DEBUG: Checking required files before Ansible execution" + if [[ ! -f "install.yaml" ]]; then + log_message "ERROR: install.yaml not found in $VDI_SETUP_DIR" + update_deployment_status "available" + return 1 + fi + if [[ ! -f "vars.yaml" ]]; then + log_message "ERROR: vars.yaml not found in $VDI_SETUP_DIR" + update_deployment_status "available" + return 1 + fi + + log_message "DEBUG: Running Ansible playbook from $(pwd)" + log_message "DEBUG: Command: ansible-playbook install.yaml --connection=local --inventory=localhost, --limit=localhost --extra-vars=@vars.yaml --extra-vars=debug=true" + + # Run Ansible playbook with re-configuration + ansible-playbook install.yaml \ + --connection=local \ + --inventory=localhost, \ + --limit=localhost \ + --extra-vars="@vars.yaml" \ + --extra-vars="debug=true" \ + >> "$ANSIBLE_LOG" 2>&1 + + local exit_code=$? + log_message "DEBUG: Ansible exit code: $exit_code" + + if [[ $exit_code -eq 0 ]]; then + log_message "VDI re-configuration completed successfully" + update_deployment_status "available" + else + log_message "ERROR: VDI re-configuration failed with exit code $exit_code" + log_message "Check $ANSIBLE_LOG for details" + # Show last few lines of Ansible log for debugging + if [[ -f "$ANSIBLE_LOG" ]]; then + log_message "DEBUG: Last 10 lines of Ansible log:" + tail -10 "$ANSIBLE_LOG" | while read line; do + log_message "DEBUG: $line" + done + fi + update_deployment_status "available" + fi + + return $exit_code +} + +# Function to update deployment status +update_deployment_status() { + local status="$1" + local instance_name=$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/name") + local zone=$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/zone" | sed 's/.*\/\([^\/]*\)$/\1/') + local project_id=$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/project/project-id") + + if [[ -n "$instance_name" && -n "$zone" && -n "$project_id" ]]; then + log_message "Updating deployment status to: $status" + + # Retry logic for metadata updates to handle fingerprint conflicts + local max_retries=3 + local retry_count=0 + local success=false + + while [[ $retry_count -lt $max_retries && "$success" == "false" ]]; do + if gcloud compute instances add-metadata "$instance_name" \ + --zone="$zone" \ + --project="$project_id" \ + --metadata="deployment-status=$status" \ + --quiet 2>/dev/null; then + success=true + log_message "Successfully updated deployment status to: $status" + else + retry_count=$((retry_count + 1)) + if [[ $retry_count -lt $max_retries ]]; then + log_message "WARNING: Failed to update deployment status (attempt $retry_count/$max_retries), retrying in 2 seconds..." + sleep 2 + else + log_message "WARNING: Failed to update deployment status after $max_retries attempts" + fi + fi + done + else + log_message "WARNING: Could not determine instance metadata for status update" + fi +} + +# Function to run monitoring loop +run_monitoring_loop() { + log_message "Starting VDI monitoring loop" + + # Initialize file tracking + initialize_file_tracking + + # Always refresh file tracking on startup to capture current bucket state + log_message "Refreshing file tracking to capture current bucket state..." + + # Debug: Check bucket name and tracking files + log_message "DEBUG: Bucket name: '$BUCKET_NAME'" + log_message "DEBUG: Checking existing tracking files:" + log_message "DEBUG: .current_vars_file exists: $([[ -f "/opt/vdi-setup/.current_vars_file" ]] && echo 'yes' || echo 'no')" + log_message "DEBUG: .current_install_file exists: $([[ -f "/opt/vdi-setup/.current_install_file" ]] && echo 'yes' || echo 'no')" + log_message "DEBUG: .current_roles_file exists: $([[ -f "/opt/vdi-setup/.current_roles_file" ]] && echo 'yes' || echo 'no')" + + # Update file tracking with current bucket state + local current_vars_file=$(get_current_file_name "vars.yaml") + log_message "DEBUG: Current vars file from bucket: '$current_vars_file'" + if [[ -n "$current_vars_file" ]]; then + store_file_name "vars" "$current_vars_file" + log_message "Updated vars file tracking: $current_vars_file" + else + log_message "WARNING: No vars.yaml file found in bucket" + fi + + local current_install_file=$(get_current_file_name "install.yaml") + log_message "DEBUG: Current install file from bucket: '$current_install_file'" + if [[ -n "$current_install_file" ]]; then + store_file_name "install" "$current_install_file" + log_message "Updated install file tracking: $current_install_file" + else + log_message "WARNING: No install.yaml file found in bucket" + fi + + local current_roles_file=$(get_current_file_name "roles.tar.gz") + log_message "DEBUG: Current roles file from bucket: '$current_roles_file'" + if [[ -n "$current_roles_file" ]]; then + store_file_name "roles" "$current_roles_file" + log_message "Updated roles file tracking: $current_roles_file" + else + log_message "WARNING: No roles.tar.gz file found in bucket" + fi + + log_message "File tracking refresh complete" + + # Cooldown variable to prevent rapid reconfigurations + local last_reconfig_time=0 + + # Check for changes immediately after initialization + log_message "Performing initial change detection check..." + + # Debug: Show current file tracking state + log_message "DEBUG: Current file tracking state:" + local stored_vars=$(get_stored_file_name 'vars') + local stored_install=$(get_stored_file_name 'install') + local stored_roles=$(get_stored_file_name 'roles') + log_message "DEBUG: vars.yaml: '$stored_vars'" + log_message "DEBUG: install.yaml: '$stored_install'" + log_message "DEBUG: roles.tar.gz: '$stored_roles'" + + if check_config_changed; then + log_message "Configuration changes detected during initialization - triggering reconfiguration" + if trigger_reconfig; then + last_reconfig_time=$(date +%s) + rm -f "/opt/vdi-setup/.pending_changes" + fi + else + log_message "No configuration changes detected during initialization" + fi + + # Main monitoring loop + local check_count=0 + while true; do + local current_time=$(date +%s) + check_count=$((check_count + 1)) + + # Log periodic check (every 5th check to avoid log spam) + if [[ $((check_count % 5)) -eq 0 ]]; then + log_message "Periodic check #${check_count} - checking for configuration changes..." + fi + + if check_config_changed; then + # Check if enough time has passed since last reconfiguration + local time_since_last_reconfig=$((current_time - last_reconfig_time)) + + if [[ $time_since_last_reconfig -ge $RECONFIG_COOLDOWN ]]; then + log_message "Configuration changes detected - triggering reconfiguration" + if trigger_reconfig; then + last_reconfig_time=$current_time + rm -f "/opt/vdi-setup/.pending_changes" + fi + else + log_message "Changes detected but cooldown active (${time_since_last_reconfig}s/${RECONFIG_COOLDOWN}s) - waiting for cooldown to expire" + fi + else + # Log when no changes detected (every 5th check) + if [[ $((check_count % 5)) -eq 0 ]]; then + log_message "No configuration changes detected in check #${check_count}" + fi + fi + + # Sleep for configured interval before next check + sleep {{ vdi_monitor_check_interval | default(60) }} + done +} \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-file.sh.j2 b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-file.sh.j2 new file mode 100644 index 0000000000..88a66cf89b --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-file.sh.j2 @@ -0,0 +1,183 @@ +#!/bin/bash + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# VDI Monitor File Module +# Contains file change detection and synchronization functions + +# Function to get GCS file hash +get_gcs_hash() { + local gcs_path="$1" + log_message_quiet "DEBUG: Getting GCS hash for: $gcs_path" + local hash_result=$(gcloud storage hash "$gcs_path" --skip-crc32c 2>&1) + local exit_code=$? + if [[ $exit_code -eq 0 ]]; then + # Extract MD5 hash from output (format: "MD5 hash: ") + local md5_hash=$(echo "$hash_result" | grep "MD5 hash:" | awk '{print $3}') + if [[ -n "$md5_hash" ]]; then + log_message_quiet "DEBUG: GCS hash result: $md5_hash" + echo "$md5_hash" + else + log_message_quiet "ERROR: Could not extract MD5 hash from output: $hash_result" + echo "INVALID-HASH" + fi + else + log_message_quiet "ERROR: Failed to get GCS hash for $gcs_path: $hash_result (exit code: $exit_code)" + echo "INVALID-HASH" + fi +} + +# Function to get local file hash +get_local_hash() { + local local_path="$1" + log_message_quiet "DEBUG: Getting local hash for: $local_path" + if [[ -f "$local_path" ]]; then + local hash_result=$(gcloud storage hash "$local_path" --skip-crc32c 2>&1) + local exit_code=$? + if [[ $exit_code -eq 0 ]]; then + # Extract MD5 hash from output (format: "MD5 hash: ") + local md5_hash=$(echo "$hash_result" | grep "MD5 hash:" | awk '{print $3}') + if [[ -n "$md5_hash" ]]; then + log_message_quiet "DEBUG: Local hash result: $md5_hash" + echo "$md5_hash" + else + log_message_quiet "ERROR: Could not extract MD5 hash from output: $hash_result" + echo "INVALID-HASH" + fi + else + log_message_quiet "ERROR: Failed to get local hash for $local_path: $hash_result (exit code: $exit_code)" + echo "INVALID-HASH" + fi + else + log_message_quiet "DEBUG: Local file does not exist: $local_path" + echo "INVALID-HASH" + fi +} + +# Function to get current file name from bucket (with hash suffix) +get_current_file_name() { + local base_name="$1" + local pattern="gs://$BUCKET_NAME/$base_name-*" + local current_file=$(gsutil ls "$pattern" 2>/dev/null | head -1) + if [[ -n "$current_file" ]]; then + # Extract just the filename from the full path + basename "$current_file" + else + echo "" + fi +} + +# Function to get stored file name +get_stored_file_name() { + local file_type="$1" + local stored_file="/opt/vdi-setup/.current_${file_type}_file" + if [[ -f "$stored_file" ]]; then + cat "$stored_file" + else + echo "" + fi +} + +# Function to store current file name +store_file_name() { + local file_type="$1" + local file_name="$2" + echo "$file_name" > "/opt/vdi-setup/.current_${file_type}_file" +} + +# Function to sync changed files from bucket +sync_changed_files() { + log_message "Syncing changed files from bucket: gs://$BUCKET_NAME" + local sync_needed=false + + # Check and sync vars.yaml + local current_vars_file=$(get_current_file_name "vars.yaml") + local stored_vars_file=$(get_stored_file_name "vars") + + if [[ "$current_vars_file" != "$stored_vars_file" ]]; then + log_message "Syncing vars.yaml: $current_vars_file" + gsutil cp "gs://$BUCKET_NAME/$current_vars_file" "$VDI_SETUP_DIR/vars.yaml" + store_file_name "vars" "$current_vars_file" + sync_needed=true + fi + + # Check and sync install.yaml + local current_install_file=$(get_current_file_name "install.yaml") + local stored_install_file=$(get_stored_file_name "install") + + if [[ "$current_install_file" != "$stored_install_file" ]]; then + log_message "Syncing install.yaml: $current_install_file" + gsutil cp "gs://$BUCKET_NAME/$current_install_file" "$VDI_SETUP_DIR/install.yaml" + store_file_name "install" "$current_install_file" + sync_needed=true + fi + + # Check and sync roles.tar.gz and unpack if needed + local current_roles_file=$(get_current_file_name "roles.tar.gz") + local stored_roles_file=$(get_stored_file_name "roles") + + if [[ "$current_roles_file" != "$stored_roles_file" ]]; then + log_message "Syncing roles.tar.gz: $current_roles_file" + gsutil cp "gs://$BUCKET_NAME/$current_roles_file" "$VDI_SETUP_DIR/roles.tar.gz" + store_file_name "roles" "$current_roles_file" + + # Unpack roles + log_message "Unpacking updated roles" + tar xzf "$VDI_SETUP_DIR/roles.tar.gz" -C "$VDI_SETUP_DIR/" + sync_needed=true + fi + + if [[ "$sync_needed" == "true" ]]; then + log_message "Successfully synced changed files from bucket" + return 0 + else + log_message "No files needed syncing" + return 1 + fi +} + +# Function to initialize file tracking +initialize_file_tracking() { + log_message "Initializing file tracking" + + # Initialize vars.yaml tracking + if [[ ! -f "/opt/vdi-setup/.current_vars_file" ]]; then + local current_vars_file=$(get_current_file_name "vars.yaml") + if [[ -n "$current_vars_file" ]]; then + store_file_name "vars" "$current_vars_file" + log_message "Initialized vars file tracking: $current_vars_file" + fi + fi + + # Initialize install.yaml tracking + if [[ ! -f "/opt/vdi-setup/.current_install_file" ]]; then + local current_install_file=$(get_current_file_name "install.yaml") + if [[ -n "$current_install_file" ]]; then + store_file_name "install" "$current_install_file" + log_message "Initialized install file tracking: $current_install_file" + fi + fi + + # Initialize roles.tar.gz tracking + if [[ ! -f "/opt/vdi-setup/.current_roles_file" ]]; then + local current_roles_file=$(get_current_file_name "roles.tar.gz") + if [[ -n "$current_roles_file" ]]; then + store_file_name "roles" "$current_roles_file" + log_message "Initialized roles file tracking: $current_roles_file" + fi + fi + + log_message "File tracking initialization complete" +} \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-test.sh.j2 b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-test.sh.j2 new file mode 100644 index 0000000000..c454c554d1 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-test.sh.j2 @@ -0,0 +1,141 @@ +#!/bin/bash + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# VDI Monitor Testing Module +# Contains testing and diagnostic functions + +# Function to run comprehensive tests +run_tests() { + log_message "VDI Configuration Monitor - Test Mode" + log_message "Testing GCS hash detection functionality..." + + # Test 1: Check if gcloud is available + log_message "Test 1: Checking gcloud availability..." + if command -v gcloud >/dev/null 2>&1; then + log_message "SUCCESS: gcloud is available" + gcloud --version | head -1 + else + log_message "ERROR: gcloud is not available" + exit 1 + fi + + # Test 2: Check if we can access the bucket + log_message "Test 2: Checking bucket access..." + if gsutil ls "gs://$BUCKET_NAME/" >/dev/null 2>&1; then + log_message "SUCCESS: Can access bucket gs://$BUCKET_NAME/" + else + log_message "ERROR: Cannot access bucket gs://$BUCKET_NAME/" + exit 1 + fi + + # Test 3: List bucket contents + log_message "Test 3: Listing bucket contents..." + gsutil ls "gs://$BUCKET_NAME/" + + # Test 4: Test file detection + log_message "Test 4: Testing file detection..." + local test_vars_file=$(get_current_file_name "vars.yaml") + local test_install_file=$(get_current_file_name "install.yaml") + local test_roles_file=$(get_current_file_name "roles.tar.gz") + + if [[ -n "$test_vars_file" && -n "$test_install_file" && -n "$test_roles_file" ]]; then + log_message "SUCCESS: File detection works" + log_message "Current files:" + log_message " vars.yaml: $test_vars_file" + log_message " install.yaml: $test_install_file" + log_message " roles.tar.gz: $test_roles_file" + else + log_message "ERROR: File detection failed" + log_message " vars.yaml: $test_vars_file" + log_message " install.yaml: $test_install_file" + log_message " roles.tar.gz: $test_roles_file" + exit 1 + fi + + # Test 5: Test reset password flag detection + log_message "Test 5: Testing reset password flag detection..." + if check_reset_password_flags; then + log_message "SUCCESS: Reset password flag detected" + else + log_message "INFO: No reset password flags found" + fi + + # Test 6: Check if configuration change detection works + log_message "Test 6: Testing configuration change detection..." + + # Check if system is properly initialized + if ! is_system_initialized; then + log_message "WARNING: System not fully initialized - initializing now..." + + # Initialize file tracking + local current_vars_file=$(get_current_file_name "vars.yaml") + if [[ -n "$current_vars_file" ]]; then + store_file_name "vars" "$current_vars_file" + fi + + local current_install_file=$(get_current_file_name "install.yaml") + if [[ -n "$current_install_file" ]]; then + store_file_name "install" "$current_install_file" + fi + + local current_roles_file=$(get_current_file_name "roles.tar.gz") + if [[ -n "$current_roles_file" ]]; then + store_file_name "roles" "$current_roles_file" + fi + + log_message "System initialization complete" + fi + + # Now test change detection + if check_config_changed; then + log_message "SUCCESS: Configuration change detected" + exit 0 + else + log_message "INFO: No configuration changes detected" + exit 0 + fi +} + +# Function to run diagnostics +run_diagnostics() { + log_message "VDI Monitor Diagnostics" + + # Environment information + log_message "Environment:" + log_message " VDI_SETUP_DIR: $VDI_SETUP_DIR" + log_message " BUCKET_NAME: $BUCKET_NAME" + log_message " MONITOR_LOG: $MONITOR_LOG" + log_message " ANSIBLE_LOG: $ANSIBLE_LOG" + + # Check file existence + log_message "File Status:" + log_message " vars.yaml exists: $([[ -f "$VDI_SETUP_DIR/vars.yaml" ]] && echo 'yes' || echo 'no')" + log_message " install.yaml exists: $([[ -f "$VDI_SETUP_DIR/install.yaml" ]] && echo 'yes' || echo 'no')" + log_message " roles directory exists: $([[ -d "$VDI_SETUP_DIR/roles" ]] && echo 'yes' || echo 'no')" + + # Check tracking files + log_message "Tracking Files:" + log_message " vars tracking: $([[ -f "/opt/vdi-setup/.current_vars_file" ]] && echo 'yes' || echo 'no')" + log_message " install tracking: $([[ -f "/opt/vdi-setup/.current_install_file" ]] && echo 'yes' || echo 'no')" + log_message " roles tracking: $([[ -f "/opt/vdi-setup/.current_roles_file" ]] && echo 'yes' || echo 'no')" + + # System status + log_message "System Status:" + log_message " Initialized: $(is_system_initialized && echo 'yes' || echo 'no')" + log_message " gcloud available: $(command -v gcloud >/dev/null 2>&1 && echo 'yes' || echo 'no')" + log_message " gsutil available: $(command -v gsutil >/dev/null 2>&1 && echo 'yes' || echo 'no')" + log_message " ansible-playbook available: $(command -v ansible-playbook >/dev/null 2>&1 && echo 'yes' || echo 'no')" +} \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/templates/vdi-monitor.service.j2 b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor.service.j2 similarity index 100% rename from community/modules/scripts/vdi-setup/roles/lock_manager/templates/vdi-monitor.service.j2 rename to community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor.service.j2 diff --git a/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor.sh.j2 b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor.sh.j2 new file mode 100644 index 0000000000..23c6b7d81f --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor.sh.j2 @@ -0,0 +1,132 @@ +#!/bin/bash + +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# VDI Configuration Monitor - Main Orchestrator +# Sources all modules and provides the main entry point + +# Source all modules +source /opt/vdi-setup/vdi-monitor-lib/config.sh +source /opt/vdi-setup/vdi-monitor-lib/file-monitor.sh +source /opt/vdi-setup/vdi-monitor-lib/change-detector.sh +source /opt/vdi-setup/vdi-monitor-lib/test-utils.sh + +# Function to update deployment status +update_deployment_status() { + local status="$1" + local instance_name=$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/name") + local zone=$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/zone" | sed 's/.*\/\([^\/]*\)$/\1/') + local project_id=$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/project/project-id") + + if [[ -n "$instance_name" && -n "$zone" && -n "$project_id" ]]; then + log_message "Updating deployment status to: $status" + + # Retry logic for metadata updates to handle fingerprint conflicts + local max_retries=3 + local retry_count=0 + local success=false + + while [[ $retry_count -lt $max_retries && "$success" == "false" ]]; do + if gcloud compute instances add-metadata "$instance_name" \ + --zone="$zone" \ + --project="$project_id" \ + --metadata="deployment-status=$status" \ + --quiet 2>/dev/null; then + success=true + log_message "Successfully updated deployment status to: $status" + else + retry_count=$((retry_count + 1)) + if [[ $retry_count -lt $max_retries ]]; then + log_message "WARNING: Failed to update deployment status (attempt $retry_count/$max_retries), retrying in 2 seconds..." + sleep 2 + else + log_message "WARNING: Failed to update deployment status after $max_retries attempts" + fi + fi + done + else + log_message "WARNING: Could not determine instance metadata for status update" + fi +} + +# Main execution +main() { + # Parse command line arguments + while [[ $# -gt 0 ]]; do + case $1 in + --test) + run_tests + exit 0 + ;; + --diagnostics) + run_diagnostics + exit 0 + ;; + --help|-h) + show_help + exit 0 + ;; + *) + echo "Unknown option: $1" + show_help + exit 1 + ;; + esac + done + + # Initialize monitoring system + initialize_monitoring + + # Set initial deployment status to available + update_deployment_status "available" + + # Run the monitoring loop + run_monitoring_loop +} + +# Entry point +case "${1:-}" in + --test) + # Initialize for testing + initialize_monitoring + run_tests + ;; + --diagnostics) + # Initialize for diagnostics + initialize_monitoring + run_diagnostics + ;; + --help|-h) + echo "VDI Configuration Monitor" + echo "" + echo "Usage: $0 [OPTION]" + echo "" + echo "Options:" + echo " --test Run comprehensive tests" + echo " --diagnostics Run system diagnostics" + echo " --help, -h Show this help message" + echo " (no args) Start monitoring" + echo "" + echo "The VDI Configuration Monitor automatically detects changes in:" + echo " - Configuration files (vars.yaml, install.yaml, roles.tar.gz)" + echo " - Password reset flags" + echo "" + echo "When changes are detected, it triggers VDI reconfiguration." + ;; + *) + # Default: start monitoring + main + ;; +esac diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml index c5f6c994ab..27bca2d727 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml @@ -13,16 +13,33 @@ # limitations under the License. --- -- name: Ensure PostgreSQL data & init directories exist - ansible.builtin.file: - path: "{{ item }}" - state: directory - owner: root - group: root - mode: '0755' - loop: - - /opt/guacamole-db/data - - /opt/guacamole-db/initdb +# ============================================================================= +# EARLY PASSWORD RESET DETECTION +# ============================================================================= + +- name: Get users that need password reset (only on existing deployments) + ansible.builtin.set_fact: + users_needing_reset: "{{ vdi_users_updated | selectattr('reset_password', 'defined') | selectattr('reset_password', 'equalto', true) | list }}" + when: not (is_fresh_deployment | default(false)) + +- name: Set empty users_needing_reset for fresh deployments + ansible.builtin.set_fact: + users_needing_reset: [] + when: is_fresh_deployment | default(false) + +- name: Debug password reset detection + ansible.builtin.debug: + msg: | + Password reset detection: + - is_fresh_deployment: {{ is_fresh_deployment | default(false) }} + - users_needing_reset: {{ users_needing_reset | length }} + - users: {{ users_needing_reset | map(attribute='username') | list }} + - user_details: {{ users_needing_reset | map('combine', {'password_length': item.password | length if item.password is defined else 'N/A'}) | list }} + when: debug | default(false) + +# ============================================================================= +# DOCKER SETUP (IDEMPOTENT) +# ============================================================================= - name: Check Docker service status ansible.builtin.service_facts: @@ -46,120 +63,70 @@ delay: 2 until: docker_test.rc == 0 -- name: Enable pgcrypto extension for initdb - ansible.builtin.copy: - dest: /opt/guacamole-db/initdb/00-enable-pgcrypto.sql - owner: root - group: root - mode: '0644' - content: | - -- Make digest(), hmac(), etc. available - CREATE EXTENSION IF NOT EXISTS pgcrypto; - -- name: Template per-user Guacamole user SQL - ansible.builtin.template: - src: guacamole/user_bootstrap.sql.j2 - dest: "/opt/guacamole-db/initdb/02-{{ item.username }}_user.sql" - owner: root - group: root - mode: '0644' - loop: "{{ vdi_users_updated }}" - loop_control: - loop_var: item - vars: - username: "{{ item.username }}" - vdi_password: "{{ item.password }}" - -- name: Template per-user SQL for Guacamole connections - ansible.builtin.template: - src: guacamole/connection_bootstrap.sql.j2 - dest: "/opt/guacamole-db/initdb/03-{{ item.username }}_connection.sql" - owner: root - group: root - mode: '0644' - loop: "{{ vdi_users_updated }}" - loop_control: - loop_var: item - vars: - port: "{{ item.port }}" - username: "{{ item.username }}" - vnc_password: "{{ item.vncserver_password }}" - -- name: Template user cleanup SQL for removed users - ansible.builtin.template: - src: guacamole/user_cleanup.sql.j2 - dest: "/opt/guacamole-db/initdb/04-user_cleanup.sql" - owner: root - group: root - mode: '0644' - vars: - vdi_users_updated: "{{ vdi_users_updated }}" - -- name: Generate combined initdb.sql +- name: Check if Docker images exist ansible.builtin.shell: | - docker run --rm \ - -v /opt/guacamole-db/initdb:/opt/guacamole-db/initdb \ - guacamole/guacamole:latest \ - /opt/guacamole/bin/initdb.sh --postgresql \ - > /opt/guacamole-db/initdb/01-initdb.sql - args: - creates: /opt/guacamole-db/initdb/01-initdb.sql - -- name: Update guacadmin password hash in 01-initdb.sql - ansible.builtin.replace: - path: /opt/guacamole-db/initdb/01-initdb.sql - regexp: "decode\\('CA458A7D494E3BE824F5E1E175A1556C0F8EEF2C2D7DF3633BEC4A29C4411960', 'hex'\\)" - replace: "decode('{{ webapp_admin_hash }}', 'hex')" - -- name: Remove salt from guacadmin in 01-initdb.sql - ansible.builtin.replace: - path: /opt/guacamole-db/initdb/01-initdb.sql - regexp: "decode\\('FE24ADC5E11E2B25288D1704ABE67A79E342ECC26064CE69C5B3177795A82264', 'hex'\\)" - replace: "NULL" - -- name: Pull required Docker images + docker images | grep -E "postgres[[:space:]]+latest" && echo "postgres:latest" || echo "" + docker images | grep -E "guacamole/guacd[[:space:]]+latest" && echo "guacamole/guacd:latest" || echo "" + docker images | grep -E "guacamole/guacamole[[:space:]]+latest" && echo "guacamole/guacamole:latest" || echo "" + register: existing_images + changed_when: false + +- name: Pull required Docker images (only if missing) ansible.builtin.command: docker pull {{ item }} loop: - postgres:latest - guacamole/guacd:latest - guacamole/guacamole:latest + when: item not in existing_images.stdout -- name: Create a dedicated Docker network +- name: Check if Docker network exists + ansible.builtin.shell: docker network ls | grep -E "guac_net" || echo "" + register: network_exists + changed_when: false + failed_when: false + +- name: Create a dedicated Docker network (only if missing) ansible.builtin.command: docker network create guac_net register: network_create failed_when: - network_create.rc != 0 - "'already exists' not in network_create.stderr" + when: "'guac_net' not in network_exists.stdout" -- name: Start Guacamole PostgreSQL container - ansible.builtin.command: > - docker run -d --name guac_db --network guac_net - -e POSTGRES_USER=guacamole_db - -e POSTGRES_PASSWORD={{ database_password }} - -e POSTGRES_DB=guacamole_db - -v /opt/guacamole-db/data:/var/lib/postgresql/data - -v /opt/guacamole-db/initdb:/docker-entrypoint-initdb.d - --restart always - postgres:latest - register: postgres_start - failed_when: - - postgres_start.rc != 0 - - "'already in use' not in postgres_start.stderr" +# ============================================================================= +# CONTAINER STATUS CHECKING +# ============================================================================= -- name: Wait for PostgreSQL container to be ready - ansible.builtin.wait_for: - host: 127.0.0.1 - port: 5432 - delay: 2 - timeout: 60 - ignore_errors: true +- name: Check if containers are already running + ansible.builtin.shell: docker ps | grep -E "(guac_db|guacd|guac_app)" || echo "" + register: running_containers + changed_when: false + failed_when: false -- name: Wait for PostgreSQL via Docker exec - ansible.builtin.command: docker exec guac_db pg_isready -U guacamole_db - register: postgres_ready - until: postgres_ready.rc == 0 - retries: 30 - delay: 2 +- name: Debug container status + ansible.builtin.debug: + msg: | + Container status check: + - running_containers: {{ running_containers.stdout_lines }} + - guac_db running: {{ 'guac_db' in running_containers.stdout }} + - guacd running: {{ 'guacd' in running_containers.stdout }} + - guac_app running: {{ 'guac_app' in running_containers.stdout }} + when: debug | default(false) + +# ============================================================================= +# DATABASE INITIALIZATION LOGIC +# ============================================================================= + +- name: Ensure PostgreSQL data & init directories exist + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: '0755' + loop: + - /opt/guacamole-db/data + - /opt/guacamole-db/initdb - name: Check if database needs re-initialization due to user changes ansible.builtin.command: > @@ -167,45 +134,131 @@ -c "SELECT COUNT(*) FROM guacamole_entity WHERE type='USER' AND name != 'guacadmin';" register: existing_users_check changed_when: false + when: "'guac_db' in running_containers.stdout" -- name: Debug user configuration status +- name: Debug user configuration status (basic info) ansible.builtin.debug: msg: | - Database user check: {{ existing_users_check.stdout.strip() }} + Database user check: N/A (no existing database) Current users in config: {{ vdi_users_updated | length }} Lock file exists: {{ lock_file_stat.stat.exists }} User secrets hash changed: {{ vdi_setup_status.user_secrets_status.user_secrets_hash != current_user_secrets_hash if lock_file_stat.stat.exists else 'N/A' }} User secrets hash (stored): {{ vdi_setup_status.user_secrets_status.user_secrets_hash if lock_file_stat.stat.exists else 'N/A' }} User secrets hash (current): {{ current_user_secrets_hash }} + when: debug | default(false) and not (existing_users_check is defined) - name: Check if user configuration has changed ansible.builtin.set_fact: - users_changed: "{{ not lock_file_stat.stat.exists or vdi_setup_status.user_secrets_status.user_secrets_hash != current_user_secrets_hash }}" + users_changed: "{{ lock_file_stat.stat.exists and vdi_setup_status.user_secrets_status.user_secrets_hash is defined and vdi_setup_status.user_secrets_status.user_secrets_hash != 'none' and vdi_setup_status.user_secrets_status.user_secrets_hash != current_user_secrets_hash }}" + +- name: Check if database needs initialization + ansible.builtin.set_fact: + db_needs_init: "{{ force_rerun | default(false) or not lock_file_stat.stat.exists or (lock_file_stat.stat.exists and vdi_setup_status.user_secrets_status.user_secrets_hash is not defined) or (lock_file_stat.stat.exists and vdi_setup_status.user_secrets_status.user_secrets_hash == 'none') }}" - name: Debug user change detection ansible.builtin.debug: - msg: "Users changed: {{ users_changed }}" - -- name: Re-initialize database if users have changed + msg: | + User change detection: + - users_changed: {{ users_changed }} + - db_needs_init: {{ db_needs_init }} + - will_reinitialize: {{ users_changed or db_needs_init }} + - users_needing_reset: {{ users_needing_reset | length }} + when: debug | default(false) + +# ============================================================================= +# DATABASE RE-INITIALIZATION (ONLY WHEN NECESSARY) +# ============================================================================= + +- name: Re-initialize database if database needs initialization block: - - name: Stop Guacamole containers - ansible.builtin.command: docker stop guac_app guacd + - name: Enable pgcrypto extension for initdb + ansible.builtin.copy: + dest: /opt/guacamole-db/initdb/00-enable-pgcrypto.sql + owner: root + group: root + mode: '0644' + content: | + -- Make digest(), hmac(), etc. available + CREATE EXTENSION IF NOT EXISTS pgcrypto; + + - name: Template per-user Guacamole user SQL + ansible.builtin.template: + src: guacamole/user_bootstrap.sql.j2 + dest: "/opt/guacamole-db/initdb/02-{{ item.username }}_user.sql" + owner: root + group: root + mode: '0644' + loop: "{{ vdi_users_updated }}" + loop_control: + loop_var: item + vars: + username: "{{ item.username }}" + vdi_password: "{{ item.password }}" + + - name: Template per-user SQL for Guacamole connections + ansible.builtin.template: + src: guacamole/connection_bootstrap.sql.j2 + dest: "/opt/guacamole-db/initdb/03-{{ item.username }}_connection.sql" + owner: root + group: root + mode: '0644' + loop: "{{ vdi_users_updated }}" + loop_control: + loop_var: item + vars: + port: "{{ item.port }}" + username: "{{ item.username }}" + vnc_password: "{{ item.vncserver_password }}" + + - name: Generate combined initdb.sql + ansible.builtin.shell: | + docker run --rm \ + -v /opt/guacamole-db/initdb:/opt/guacamole-db/initdb \ + guacamole/guacamole:latest \ + /opt/guacamole/bin/initdb.sh --postgresql \ + > /opt/guacamole-db/initdb/01-initdb.sql + args: + creates: /opt/guacamole-db/initdb/01-initdb.sql + + - name: Update guacadmin password hash in 01-initdb.sql + ansible.builtin.replace: + path: /opt/guacamole-db/initdb/01-initdb.sql + regexp: "decode\\('CA458A7D494E3BE824F5E1E175A1556C0F8EEF2C2D7DF3633BEC4A29C4411960', 'hex'\\)" + replace: "decode('{{ webapp_admin_hash }}', 'hex')" + + - name: Remove salt from guacadmin in 01-initdb.sql + ansible.builtin.replace: + path: /opt/guacamole-db/initdb/01-initdb.sql + regexp: "decode\\('FE24ADC5E11E2B25288D1704ABE67A79E342ECC26064CE69C5B3177795A82264', 'hex'\\)" + replace: "NULL" + + - name: Debug webapp admin password usage + ansible.builtin.debug: + msg: | + Webapp admin password usage (vdi_tool): + - password_provided_by_secret_manager: {{ webapp_admin_password is defined }} + - password_length: {{ webapp_admin_password | length if webapp_admin_password is defined else 'N/A' }} + - hash_computed: {{ webapp_admin_hash is defined }} + when: debug | default(false) + + - name: Stop and remove existing containers + ansible.builtin.command: docker stop {{ item }} + loop: + - guac_app + - guacd + - guac_db ignore_errors: true - - name: Remove Guacamole containers - ansible.builtin.command: docker rm guac_app guacd - ignore_errors: true - - - name: Stop and remove PostgreSQL container - ansible.builtin.command: docker stop guac_db - ignore_errors: true - - - name: Remove PostgreSQL container - ansible.builtin.command: docker rm guac_db + - name: Remove existing containers + ansible.builtin.command: docker rm {{ item }} + loop: + - guac_app + - guacd + - guac_db ignore_errors: true - name: Backup existing database data - ansible.builtin.command: > + ansible.builtin.shell: > tar -czf /opt/guacamole-db/backup-$(date +%Y%m%d-%H%M%S).tar.gz -C /opt/guacamole-db data/ ignore_errors: true @@ -236,9 +289,112 @@ retries: 30 delay: 2 - when: users_changed + - name: Check if Guacamole tables exist + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_name LIKE 'guacamole_%';" + register: table_check + changed_when: false + + - name: Debug table check + ansible.builtin.debug: + msg: | + Guacamole tables found: + {{ table_check.stdout }} + when: debug | default(false) + + - name: Check if connections were created + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c + "SELECT connection_name, protocol FROM guacamole_connection WHERE connection_name NOT LIKE '%SSH';" + register: connection_check + changed_when: false + + - name: Debug connection check + ansible.builtin.debug: + msg: | + VNC Connections found: + {{ connection_check.stdout }} + when: debug | default(false) + + when: db_needs_init + +# ============================================================================= +# TARGETED PASSWORD UPDATES (NO CONTAINER RECREATION) +# ============================================================================= + +- name: Handle user changes without container recreation + block: + - name: Debug user update mode + ansible.builtin.debug: + msg: | + User changes detected - applying targeted updates (no container recreation): + - users_changed: {{ users_changed }} + - db_needs_init: {{ db_needs_init }} + - force_rerun: {{ force_rerun | default(false) }} + - users_needing_reset: {{ users_needing_reset | length }} + when: debug | default(false) + + - name: Update user passwords in database (for reset_password users) + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c "UPDATE guacamole_user SET password_salt = NULL, password_hash = digest('{{ item.password }}', 'SHA-256'), password_date = now() FROM guacamole_entity WHERE guacamole_user.entity_id = guacamole_entity.entity_id AND guacamole_entity.name = '{{ item.username }}' AND guacamole_entity.type = 'USER';" + loop: "{{ users_needing_reset }}" + when: users_needing_reset | length > 0 + + - name: Update connection ports for users + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c "UPDATE guacamole_connection_parameter SET parameter_value = '{{ item.port }}' FROM guacamole_connection WHERE guacamole_connection_parameter.connection_id = guacamole_connection.connection_id AND guacamole_connection.connection_name = '{{ item.username }}' AND guacamole_connection_parameter.parameter_name = 'port';" + loop: "{{ vdi_users_updated }}" -- name: Start guacd container + - name: Debug password updates + ansible.builtin.debug: + msg: | + Password updates applied: + {% for user in users_needing_reset %} + - {{ user.username }}: password updated + {% endfor %} + Total users needing reset: {{ users_needing_reset | length }} + when: debug | default(false) and users_needing_reset | length > 0 + + when: users_needing_reset | length > 0 and not db_needs_init + +# ============================================================================= +# CONTAINER MANAGEMENT (IDEMPOTENT) +# ============================================================================= + +- name: Wait for PostgreSQL container to be ready + ansible.builtin.wait_for: + host: 127.0.0.1 + port: 5432 + delay: 2 + timeout: 60 + ignore_errors: true + when: "'guac_db' not in running_containers.stdout" + +- name: Wait for PostgreSQL via Docker exec + ansible.builtin.command: docker exec guac_db pg_isready -U guacamole_db + register: postgres_ready + until: postgres_ready.rc == 0 + retries: 30 + delay: 2 + when: "'guac_db' not in running_containers.stdout" + +- name: Check if connections exist (non-reinitialisation case) + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c + "SELECT connection_name, protocol FROM guacamole_connection WHERE connection_name NOT LIKE '%SSH';" + register: existing_connection_check + changed_when: false + when: not db_needs_init + +- name: Debug existing connection check + ansible.builtin.debug: + msg: | + Existing VNC Connections: + {{ existing_connection_check.stdout }} + when: debug | default(false) and not db_needs_init + +- name: Start guacd container (only if not running) ansible.builtin.command: > docker run -d --name guacd --network guac_net -p 4822:4822 @@ -248,8 +404,9 @@ failed_when: - guacd_start.rc != 0 - "'already in use' not in guacd_start.stderr" + when: "'guacd' not in running_containers.stdout" -- name: Start Guacamole webapp container +- name: Start Guacamole webapp container (only if not running) ansible.builtin.command: > docker run -d --name guac_app --network guac_net -p {{ vdi_webapp_port }}:8080 @@ -267,6 +424,11 @@ failed_when: - guac_app_start.rc != 0 - "'already in use' not in guac_app_start.stderr" + when: "'guac_app' not in running_containers.stdout" + +# ============================================================================= +# VERIFICATION AND CLEANUP +# ============================================================================= - name: Debug container status before waiting for Guacamole ansible.builtin.command: docker ps @@ -276,6 +438,7 @@ - name: Show container status ansible.builtin.debug: msg: "{{ container_status.stdout }}" + when: debug | default(false) - name: Wait for Guacamole HTTP endpoint ansible.builtin.wait_for: @@ -294,6 +457,77 @@ retries: 10 delay: 3 +- name: Verify connections were created in database + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c + "SELECT connection_name, protocol FROM guacamole_connection WHERE connection_name IN ({% for user in vdi_users_updated %}'{{ user.username }}'{% if not loop.last %},{% endif %}{% endfor %},{% for user in vdi_users_updated %}'{{ user.username }} SSH'{% if not loop.last %},{% endif %}{% endfor %});" + register: connection_verification + changed_when: false + +- name: Debug connection verification + ansible.builtin.debug: + msg: | + Connection verification results: + {{ connection_verification.stdout }} + when: debug | default(false) + +- name: Verify connection permissions were created + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c + "SELECT e.name as username, c.connection_name, cp.permission + FROM guacamole_entity e + JOIN guacamole_connection_permission cp ON e.entity_id = cp.entity_id + JOIN guacamole_connection c ON cp.connection_id = c.connection_id + WHERE e.type = 'USER' AND e.name != 'guacadmin' + ORDER BY e.name, c.connection_name, cp.permission;" + register: permission_verification + changed_when: false + +- name: Debug permission verification + ansible.builtin.debug: + msg: | + Permission verification results: + {{ permission_verification.stdout }} + when: debug | default(false) + +- name: Debug fresh deployment reset logic + ansible.builtin.debug: + msg: | + Fresh deployment detected - password reset logic skipped: + - is_fresh_deployment: {{ is_fresh_deployment | default(false) }} + - users_needing_reset: [] (empty list) + - reset_password flags will be ignored + - reset_webapp_admin_password: {{ reset_webapp_admin_password | default(false) }} (will be ignored on fresh deployment) + when: debug | default(false) and is_fresh_deployment | default(false) + +- name: Update webapp admin password in database (if reset requested) + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c + "UPDATE guacamole_user SET password_salt = NULL, password_hash = decode('{{ webapp_admin_hash }}', 'hex'), password_date = now() FROM guacamole_entity WHERE guacamole_user.entity_id = guacamole_entity.entity_id AND guacamole_entity.name = 'guacadmin' AND guacamole_entity.type = 'USER';" + when: reset_webapp_admin_password | default(false) and not (is_fresh_deployment | default(false)) + +- name: Debug webapp admin password update + ansible.builtin.debug: + msg: | + Webapp admin password update: + - reset_requested: {{ reset_webapp_admin_password | default(false) }} + - is_fresh_deployment: {{ is_fresh_deployment | default(false) }} + - password_updated: {{ reset_webapp_admin_password | default(false) and not (is_fresh_deployment | default(false)) }} + - password_length: {{ webapp_admin_password | length if webapp_admin_password is defined else 'N/A' }} + when: debug | default(false) and reset_webapp_admin_password | default(false) + +- name: Restart Guacamole webapp container (if webapp admin password reset) + ansible.builtin.command: docker restart guac_app + when: reset_webapp_admin_password | default(false) and not (is_fresh_deployment | default(false)) + +- name: Wait for Guacamole webapp to be ready after restart + ansible.builtin.wait_for: + host: 127.0.0.1 + port: "{{ vdi_webapp_port }}" + delay: 5 + timeout: 60 + when: reset_webapp_admin_password | default(false) and not (is_fresh_deployment | default(false)) + - name: Obtain Guacamole API token to verify login ansible.builtin.uri: url: "http://localhost:{{ vdi_webapp_port }}/guacamole/api/tokens" diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml index 6099616b86..d89a4fe845 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml @@ -28,6 +28,24 @@ ansible.builtin.set_fact: role_should_run: "{{ lock_check_result.ansible_facts.role_should_run | default(true) }}" +- name: Check if user secrets changed + ansible.builtin.set_fact: + user_secrets_changed: "{{ lock_check_result.ansible_facts.user_secrets_changed | default(false) }}" + +- name: Override role_should_run if user secrets changed + ansible.builtin.set_fact: + role_should_run: true + when: user_secrets_changed + +- name: Debug vdi_tool role execution reason + ansible.builtin.debug: + msg: | + VDI tool role execution: + - role_should_run: {{ role_should_run }} + - user_secrets_changed: {{ user_secrets_changed }} + - execution_reason: {{ 'User configuration changed' if user_secrets_changed else 'Normal role execution' }} + when: role_should_run + # Skip all tasks if role should not run - name: Skip vdi_tool tasks if role should not run ansible.builtin.debug: diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_bootstrap.sql.j2 b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_bootstrap.sql.j2 index cc5e261d23..7685d83532 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_bootstrap.sql.j2 +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_bootstrap.sql.j2 @@ -1,13 +1,6 @@ --- VNC Connection for {{ username }} (upsert pattern) --- First, ensure the connection exists +-- VNC Connection for {{ username }} INSERT INTO guacamole_connection (connection_name, protocol) -VALUES ('{{ username }}', 'vnc') -ON CONFLICT (connection_name) DO UPDATE SET - protocol = EXCLUDED.protocol; - --- Update connection parameters (delete old ones first, then insert new ones) -DELETE FROM guacamole_connection_parameter -WHERE connection_id = (SELECT connection_id FROM guacamole_connection WHERE connection_name = '{{ username }}'); +VALUES ('{{ username }}', 'vnc'); INSERT INTO guacamole_connection_parameter (connection_id, parameter_name, parameter_value) SELECT @@ -22,11 +15,6 @@ FROM guacamole_connection, (VALUES ) AS params(param_name, param_value) WHERE connection_name = '{{ username }}'; --- Update connection permissions (delete old ones first, then insert new ones) -DELETE FROM guacamole_connection_permission -WHERE connection_id = (SELECT connection_id FROM guacamole_connection WHERE connection_name = '{{ username }}') -AND entity_id = (SELECT entity_id FROM guacamole_entity WHERE name = '{{ username }}' AND type = 'USER'); - INSERT INTO guacamole_connection_permission (entity_id, connection_id, permission) SELECT (SELECT entity_id FROM guacamole_entity WHERE name = '{{ username }}' AND type = 'USER'), @@ -34,16 +22,9 @@ SELECT perms.permission::guacamole_object_permission_type FROM (VALUES ('READ'),('UPDATE'),('DELETE'),('ADMINISTER')) AS perms(permission); --- SSH Connection for {{ username }} (upsert pattern) --- First, ensure the SSH connection exists +-- SSH Connection for {{ username }} INSERT INTO guacamole_connection (connection_name, protocol) -VALUES ('{{ username }} SSH', 'ssh') -ON CONFLICT (connection_name) DO UPDATE SET - protocol = EXCLUDED.protocol; - --- Update SSH connection parameters (delete old ones first, then insert new ones) -DELETE FROM guacamole_connection_parameter -WHERE connection_id = (SELECT connection_id FROM guacamole_connection WHERE connection_name = '{{ username }} SSH'); +VALUES ('{{ username }} SSH', 'ssh'); INSERT INTO guacamole_connection_parameter (connection_id, parameter_name, parameter_value) SELECT @@ -65,11 +46,6 @@ FROM guacamole_connection, (VALUES ) AS params(param_name, param_value) WHERE connection_name = '{{ username }} SSH'; --- Update SSH connection permissions (delete old ones first, then insert new ones) -DELETE FROM guacamole_connection_permission -WHERE connection_id = (SELECT connection_id FROM guacamole_connection WHERE connection_name = '{{ username }} SSH') -AND entity_id = (SELECT entity_id FROM guacamole_entity WHERE name = '{{ username }}' AND type = 'USER'); - INSERT INTO guacamole_connection_permission (entity_id, connection_id, permission) SELECT (SELECT entity_id FROM guacamole_entity WHERE name = '{{ username }}' AND type = 'USER'), diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/user_bootstrap.sql.j2 b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/user_bootstrap.sql.j2 index 33d3ef7dee..784d949d4e 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/user_bootstrap.sql.j2 +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/user_bootstrap.sql.j2 @@ -1,10 +1,7 @@ --- Create or update Guacamole user for {{ username }} --- First, ensure the entity exists +-- Create Guacamole user for {{ username }} INSERT INTO guacamole_entity (name, type) -VALUES ('{{ username }}', 'USER') -ON CONFLICT (name, type) DO NOTHING; +VALUES ('{{ username }}', 'USER'); --- Then, insert or update the user with the latest password INSERT INTO guacamole_user ( entity_id, password_salt, @@ -21,9 +18,4 @@ SELECT false, false FROM guacamole_entity -WHERE name = '{{ username }}' AND type = 'USER' -ON CONFLICT (entity_id) DO UPDATE SET - password_hash = EXCLUDED.password_hash, - password_date = EXCLUDED.password_date, - disabled = EXCLUDED.disabled, - expired = EXCLUDED.expired; +WHERE name = '{{ username }}' AND type = 'USER'; diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/user_cleanup.sql.j2 b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/user_cleanup.sql.j2 deleted file mode 100644 index cf323b01dc..0000000000 --- a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/user_cleanup.sql.j2 +++ /dev/null @@ -1,143 +0,0 @@ --- Cleanup script to remove users and connections that are no longer in the configuration --- This script should be run after all current users have been processed - -{% if vdi_users_updated | length > 0 %} --- Remove connections for users that no longer exist in the configuration -DELETE FROM guacamole_connection_permission -WHERE connection_id IN ( - SELECT connection_id - FROM guacamole_connection - WHERE connection_name IN ( - -- Remove VNC connections for users not in current config - SELECT connection_name - FROM guacamole_connection - WHERE protocol = 'vnc' - AND connection_name NOT IN ( - {% for user in vdi_users_updated %} - '{{ user.username }}'{% if not loop.last %},{% endif %} - {% endfor %} - ) - ) - OR connection_name IN ( - -- Remove SSH connections for users not in current config - SELECT connection_name - FROM guacamole_connection - WHERE protocol = 'ssh' - AND connection_name NOT IN ( - {% for user in vdi_users_updated %} - '{{ user.username }} SSH'{% if not loop.last %},{% endif %} - {% endfor %} - ) - ) -); - --- Remove connection parameters for connections that will be deleted -DELETE FROM guacamole_connection_parameter -WHERE connection_id IN ( - SELECT connection_id - FROM guacamole_connection - WHERE connection_name IN ( - -- Remove VNC connections for users not in current config - SELECT connection_name - FROM guacamole_connection - WHERE protocol = 'vnc' - AND connection_name NOT IN ( - {% for user in vdi_users_updated %} - '{{ user.username }}'{% if not loop.last %},{% endif %} - {% endfor %} - ) - ) - OR connection_name IN ( - -- Remove SSH connections for users not in current config - SELECT connection_name - FROM guacamole_connection - WHERE protocol = 'ssh' - AND connection_name NOT IN ( - {% for user in vdi_users_updated %} - '{{ user.username }} SSH'{% if not loop.last %},{% endif %} - {% endfor %} - ) - ) -); - --- Remove the connections themselves -DELETE FROM guacamole_connection -WHERE connection_name IN ( - -- Remove VNC connections for users not in current config - SELECT connection_name - FROM guacamole_connection - WHERE protocol = 'vnc' - AND connection_name NOT IN ( - {% for user in vdi_users_updated %} - '{{ user.username }}'{% if not loop.last %},{% endif %} - {% endfor %} - ) -) -OR connection_name IN ( - -- Remove SSH connections for users not in current config - SELECT connection_name - FROM guacamole_connection - WHERE protocol = 'ssh' - AND connection_name NOT IN ( - {% for user in vdi_users_updated %} - '{{ user.username }} SSH'{% if not loop.last %},{% endif %} - {% endfor %} - ) -); - --- Remove user permissions for users that no longer exist -DELETE FROM guacamole_connection_permission -WHERE entity_id IN ( - SELECT entity_id - FROM guacamole_entity - WHERE type = 'USER' - AND name NOT IN ( - {% for user in vdi_users_updated %} - '{{ user.username }}'{% if not loop.last %},{% endif %} - {% endfor %} - ) -); - --- Remove the users themselves -DELETE FROM guacamole_user -WHERE entity_id IN ( - SELECT entity_id - FROM guacamole_entity - WHERE type = 'USER' - AND name NOT IN ( - {% for user in vdi_users_updated %} - '{{ user.username }}'{% if not loop.last %},{% endif %} - {% endfor %} - ) -); - --- Remove the entities for users that no longer exist -DELETE FROM guacamole_entity -WHERE type = 'USER' -AND name NOT IN ( - {% for user in vdi_users_updated %} - '{{ user.username }}'{% if not loop.last %},{% endif %} - {% endfor %} -); -{% else %} --- No users in current configuration, remove all non-admin users -DELETE FROM guacamole_connection_permission -WHERE entity_id IN ( - SELECT entity_id - FROM guacamole_entity - WHERE type = 'USER' - AND name != 'guacadmin' -); - -DELETE FROM guacamole_user -WHERE entity_id IN ( - SELECT entity_id - FROM guacamole_entity - WHERE type = 'USER' - AND name != 'guacadmin' -); - -DELETE FROM guacamole_entity -WHERE type = 'USER' -AND name != 'guacadmin'; -{% endif %} \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml index 3507180468..feab71396d 100644 --- a/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml @@ -28,6 +28,24 @@ ansible.builtin.set_fact: role_should_run: "{{ lock_check_result.ansible_facts.role_should_run | default(true) }}" +- name: Check if user secrets changed + ansible.builtin.set_fact: + user_secrets_changed: "{{ lock_check_result.ansible_facts.user_secrets_changed | default(false) }}" + +- name: Override role_should_run if user secrets changed + ansible.builtin.set_fact: + role_should_run: true + when: user_secrets_changed + +- name: Debug vnc role execution reason + ansible.builtin.debug: + msg: | + VNC role execution: + - role_should_run: {{ role_should_run }} + - user_secrets_changed: {{ user_secrets_changed }} + - execution_reason: {{ 'User configuration changed' if user_secrets_changed else 'Normal role execution' }} + when: role_should_run + # Skip all tasks if role should not run - name: Skip vnc tasks if role should not run ansible.builtin.debug: diff --git a/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl b/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl index ac1de84902..91ad46ddf6 100644 --- a/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl +++ b/community/modules/scripts/vdi-setup/templates/install.yaml.tftpl @@ -3,15 +3,9 @@ - hosts: localhost become: true vars_files: - - "/tmp/vdi/vars.yaml" + - "/opt/vdi-setup/vars.yaml" tasks: - # Initialize lock manager - - name: Initialize lock manager - ansible.builtin.import_role: - name: lock_manager - tasks_from: check_lock - %{ for r in roles } - name: Execute ${r} role ansible.builtin.import_role: diff --git a/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl b/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl index 25e202546d..5b90d80ed1 100644 --- a/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl +++ b/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl @@ -1,15 +1,18 @@ # Generated by Cluster Toolkit -deployment_name: ${deployment_name} -project_id: ${project_id} -user_provision: ${user_provision} -vnc_flavor: ${vnc_flavor} -vdi_tool: ${vdi_tool} -vdi_user_group: ${vdi_user_group} -vdi_webapp_port: ${vdi_webapp_port} -vdi_resolution: ${vdi_resolution} -vdi_bucket_name: ${vdi_bucket_name} -zone: ${zone} +deployment_name: ${deployment_name} +project_id: ${project_id} +user_provision: ${user_provision} +vnc_flavor: ${vnc_flavor} +vdi_tool: ${vdi_tool} +vdi_user_group: ${vdi_user_group} +vdi_webapp_port: ${vdi_webapp_port} +vdi_resolution: ${vdi_resolution} +vdi_bucket_name: ${vdi_bucket_name} +zone: ${zone} +debug: ${debug} +reset_webapp_admin_password: ${reset_webapp_admin_password} +force_rerun: ${force_rerun} vdi_users: %{ for user in vdi_users } @@ -21,5 +24,7 @@ vdi_users: %{ if user.secret_project != null } secret_project: "${user.secret_project}" %{ endif } - +%{ if user.reset_password != null } + reset_password: ${user.reset_password} +%{ endif } %{ endfor } diff --git a/community/modules/scripts/vdi-setup/validation.tf b/community/modules/scripts/vdi-setup/validation.tf index 02c3dfc822..cd352b54cc 100644 --- a/community/modules/scripts/vdi-setup/validation.tf +++ b/community/modules/scripts/vdi-setup/validation.tf @@ -49,5 +49,14 @@ resource "terraform_data" "input_validation" { ]) error_message = "Each VDI user must have a port between 5901 and 5999 when VNC is used." } + + precondition { + condition = alltrue([ + for user in var.vdi_users : ( + user.reset_password == null || user.reset_password == true || user.reset_password == false + ) + ]) + error_message = "reset_password must be a boolean value (true/false) or null." + } } } diff --git a/community/modules/scripts/vdi-setup/variables.tf b/community/modules/scripts/vdi-setup/variables.tf index 1dd7ffd75e..ca959ba532 100644 --- a/community/modules/scripts/vdi-setup/variables.tf +++ b/community/modules/scripts/vdi-setup/variables.tf @@ -74,12 +74,13 @@ variable "vdi_webapp_port" { } variable "vdi_users" { - description = "List of VDI users to configure. Passwords are handled securely by the Ansible roles: if secret_name is provided, the password is fetched from Secret Manager; if neither password nor secret_name is provided, a random password is generated and stored in Secret Manager. If secret_project is provided, it specifies the GCP project where the secret is stored (defaults to the deployment project)." + description = "List of VDI users to configure. Passwords are handled securely by the Ansible roles: if secret_name is provided, the password is fetched from Secret Manager; if neither password nor secret_name is provided, a random password is generated and stored in Secret Manager. If secret_project is provided, it specifies the GCP project where the secret is stored (defaults to the deployment project). Set reset_password to true to trigger password regeneration for auto-generated passwords." type = list(object({ username = string port = number secret_name = optional(string) secret_project = optional(string) + reset_password = optional(bool) })) default = [] } @@ -100,3 +101,15 @@ variable "debug" { default = false description = "Enable debug mode for verbose logging during VDI setup." } + +variable "reset_webapp_admin_password" { + type = bool + default = false + description = "Force reset of the webapp admin password during reconfiguration. If true, a new password will be generated and stored in Secret Manager, even if an existing password exists." +} + +variable "force_rerun" { + type = bool + default = false + description = "Force complete container recreation and database re-initialization, bypassing all idempotency checks. Use only when troubleshooting or when the system is in a broken state." +} From 331aea48befc480df5bff748f683045a6add6a1a Mon Sep 17 00:00:00 2001 From: Scott Gordon Date: Fri, 22 Aug 2025 12:18:34 +0100 Subject: [PATCH 18/21] vdi module - minor fixes --- community/modules/scripts/vdi-setup/README.md | 104 +- community/modules/scripts/vdi-setup/main.tf | 3 +- .../vdi-setup/roles/base_os/tasks/main.yaml | 117 +- .../vdi-setup/roles/lock_manager/README.md | 150 +++ .../roles/lock_manager/tasks/check_lock.yaml | 298 +++-- .../lock_manager/tasks/cleanup_lock.yaml | 2 +- .../roles/lock_manager/tasks/create_lock.yaml | 245 ++-- .../roles/lock_manager/tasks/main.yaml | 13 +- .../lock_manager/templates/lock_file.yaml.j2 | 44 +- .../roles/secret_manager/tasks/main.yaml | 272 ++--- .../tasks/user_secret_tasks.yaml | 120 +- .../user_provision/tasks/local_users.yaml | 62 +- .../roles/user_provision/tasks/main.yaml | 60 +- .../vdi-setup/roles/vdi_monitor/README.md | 104 +- .../roles/vdi_monitor/defaults/main.yaml | 3 +- .../roles/vdi_monitor/tasks/main.yaml | 77 +- .../templates/vdi-monitor-config.sh.j2 | 4 +- .../templates/vdi-monitor-detector.sh.j2 | 6 +- .../templates/vdi-monitor-file.sh.j2 | 2 +- .../templates/vdi-monitor-test.sh.j2 | 2 +- .../roles/vdi_tool/tasks/guacamole.yaml | 1014 ++++++++++++----- .../vdi-setup/roles/vdi_tool/tasks/main.yaml | 69 +- .../guacamole/connection_bootstrap.sql.j2 | 3 +- .../guacamole/connection_port_update.sql.j2 | 5 + .../vdi-setup/roles/vnc/tasks/main.yaml | 238 +++- .../vdi-setup/roles/vnc/tasks/tigervnc.yaml | 15 + .../roles/vnc/tasks/vnc_user_config.yaml | 3 +- .../vdi-setup/templates/vars.yaml.tftpl | 1 + .../modules/scripts/vdi-setup/validation.tf | 10 + .../modules/scripts/vdi-setup/variables.tf | 8 +- .../modules/scripts/vdi-setup/versions.tf | 4 + 31 files changed, 2042 insertions(+), 1016 deletions(-) create mode 100644 community/modules/scripts/vdi-setup/roles/lock_manager/README.md create mode 100644 community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_port_update.sql.j2 diff --git a/community/modules/scripts/vdi-setup/README.md b/community/modules/scripts/vdi-setup/README.md index 928a278a43..18cd2fee32 100644 --- a/community/modules/scripts/vdi-setup/README.md +++ b/community/modules/scripts/vdi-setup/README.md @@ -1,15 +1,17 @@ ## Description -Creates a containerised Guacamole instance. Works with the Rocky, Debian and Ubuntu images shown in blueprint example below. +Creates a containerised Guacamole instance. Currently designed to work mainly with the Rocky image in the blueprint example below. Ubuntu/Debian support previously worked too but fixes may be required due to updates. ## Features - **VDI Tool Support**: Currently supports Guacamole for web-based VDI access -- **User Provisioning**: Supports local user creation with secure password management -- **VNC Integration**: Configures VNC servers for desktop access +- **User Provisioning**: Supports local user creation with secure password management and group management +- **VNC Integration**: Configures VNC servers for desktop access with port change support - **Secret Manager Integration**: Secure password storage and retrieval for VDI users and webapp admin -- **VDI Monitoring System**: Automatic monitoring and reconfiguration capabilities with password reset flags and deployment status updates +- **VDI Monitoring System**: Automatic monitoring and reconfiguration capabilities with password reset flags and deployment status updates. [More info](roles/vdi_monitor/README.md). - **Debug Mode**: Comprehensive logging when enabled via the `debug` variable +- **Port Change Support**: Handles both webapp port changes (container recreation) and user VNC port changes (SQL updates) +- **Resolution Control**: Configurable VDI resolution with browser scaling control via `vdi_resolution_locked` parameter ## Secret Manager Integration @@ -21,11 +23,31 @@ The module integrates with Google Cloud Secret Manager for secure password handl - **Reset Functionality**: Both individual user passwords (`reset_password`) and webapp admin password (`reset_webapp_admin_password`) can be forced to regenerate - **Database Updates**: Password resets update existing database records without re-initializing the entire database +## Port Change and User Group Management + +The module supports dynamic configuration changes without full redeployment: + +### **Webapp Port Changes** +- **Container Recreation**: When `vdi_webapp_port` changes, the Guacamole webapp container is recreated with the new port +- **Database Synchronization**: Database passwords are automatically synchronized during port changes +- **Service Continuity**: VNC services remain unaffected during webapp port changes + +### **User VNC Port Changes** +- **SQL Updates**: User VNC port changes are handled via database updates without container recreation +- **VNC Service Management**: VNC services are automatically stopped and restarted with new ports +- **X Server Cleanup**: Old X server processes are properly terminated before starting new ones + +### **User Group Management** +- **Dynamic Group Changes**: When `vdi_user_group` changes, users are automatically migrated to the new group +- **Group Removal**: Users are removed from the old group and added to the new group +- **Idempotent Operations**: Group changes are safe to apply multiple times + ## VDI Monitoring System The module includes a monitoring system that: -- **Deployment Status**: Updates instance metadata to reflect deployment state (`available`, `reconfiguring`) +- **Deployment Status**: Updates instance metadata to reflect deployment state (`available`, `reconfiguring`, `failed`) +- **Targeted Updates**: Performs database-only updates for password resets and user changes without container recreation - **Password Reset Flags**: Supports `reset_password` for individual users and `reset_webapp_admin_password` for the webapp admin account - **Automatic Reconfiguration**: Detects changes and triggers reconfiguration when needed - **Status Tracking**: Maintains deployment status through instance metadata @@ -74,6 +96,7 @@ deployment_groups: vdi_tool: guacamole vdi_user_group: vdiusers vdi_resolution: 1920x1080 + vdi_resolution_locked: true # Enable debug mode for verbose logging debug: true # Force reset of webapp admin password (optional) @@ -101,14 +124,13 @@ deployment_groups: source: modules/compute/vm-instance settings: instance_image: - # Several supported image families: family: hpc-rocky-linux-8 project: cloud-hpc-image-public #family: debian-11 #project: debian-cloud #family: ubuntu-2204-lts #project: ubuntu-os-cloud - name_prefix: debian + name_prefix: guacamole add_deployment_name_before_prefix: true machine_type: e2-highcpu-8 tags: ["guacamole"] @@ -156,3 +178,71 @@ After deployment, you can access the VDI in several ways: ``` - The web interface will then be accessible from http://localhost:8080/guacamole/ (for Guacamole) + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [archive](#requirement\_archive) | ~> 2.0 | +| [google](#requirement\_google) | >= 3.83 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [archive](#provider\_archive) | ~> 2.0 | +| [google](#provider\_google) | >= 3.83 | +| [random](#provider\_random) | ~> 3.0 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [startup\_script](#module\_startup\_script) | ../../../../modules/scripts/startup-script | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_storage_bucket.bucket](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | +| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [terraform_data.input_validation](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [archive_file.roles_tar](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [debug](#input\_debug) | Enable debug mode for verbose logging during VDI setup. | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | n/a | yes | +| [force\_rerun](#input\_force\_rerun) | Force complete container recreation and database re-initialization, bypassing all idempotency checks. Use only when troubleshooting or when the system is in a broken state. | `bool` | `false` | no | +| [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | +| [region](#input\_region) | Region to place bucket containing startup script. | `string` | n/a | yes | +| [reset\_webapp\_admin\_password](#input\_reset\_webapp\_admin\_password) | Force reset of the webapp admin password during reconfiguration. If true, a new password will be generated and stored in Secret Manager, even if an existing password exists. | `bool` | `false` | no | +| [user\_provision](#input\_user\_provision) | User type to create (local\_users supported. os-login to do. | `string` | `"local_users"` | no | +| [vdi\_resolution](#input\_vdi\_resolution) | Desktop resolution for VNC sessions (e.g. 1920x1080). | `string` | `"1920x1080"` | no | +| [vdi\_resolution\_locked](#input\_vdi\_resolution\_locked) | Disable resize of remote display in Guacamole connections. When true, VDI displays at native resolution without browser scaling. | `bool` | `true` | no | +| [vdi\_tool](#input\_vdi\_tool) | VDI tool to deploy (guacamole currently supported). | `string` | `"guacamole"` | no | +| [vdi\_user\_group](#input\_vdi\_user\_group) | Unix group to create/use for VDI users. | `string` | `"vdiusers"` | no | +| [vdi\_users](#input\_vdi\_users) | List of VDI users to configure. Passwords are handled securely by the Ansible roles: if secret\_name is provided, the password is fetched from Secret Manager; if neither password nor secret\_name is provided, a random password is generated and stored in Secret Manager. If secret\_project is provided, it specifies the GCP project where the secret is stored (defaults to the deployment project). Set reset\_password to true to trigger password regeneration for auto-generated passwords. | 
list(object({
    username       = string
    port           = number
    secret_name    = optional(string)
    secret_project = optional(string)
    reset_password = optional(bool)
  }))
| `[]` | no | +| [vdi\_webapp\_port](#input\_vdi\_webapp\_port) | Port to serve the Webapp interface from if applicable (note: containers will be recreated if changed) | `string` | `"8080"` | no | +| [vnc\_flavor](#input\_vnc\_flavor) | The VNC server flavor to use (tigervnc currently supported) | `string` | `"tigervnc"` | no | +| [vnc\_port\_max](#input\_vnc\_port\_max) | Maximum valid VNC port. | `number` | `5999` | no | +| [vnc\_port\_min](#input\_vnc\_port\_min) | Minimum valid VNC port. | `number` | `5901` | no | +| [zone](#input\_zone) | Zone in which the VDI instances are created. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [guacamole\_admin\_password\_secret](#output\_guacamole\_admin\_password\_secret) | The name of the Secret Manager secret containing the Guacamole admin password | +| [guacamole\_admin\_username](#output\_guacamole\_admin\_username) | The admin username for Guacamole | +| [startup\_script](#output\_startup\_script) | Combined startup script that installs VDI (VNC, Guacamole, users). | +| [vdi\_runner](#output\_vdi\_runner) | Shell runner wrapping Ansible playbook + roles (for custom-image or direct use). | +| [vdi\_user\_credentials](#output\_vdi\_user\_credentials) | Map of VDI user credentials stored in Secret Manager | + diff --git a/community/modules/scripts/vdi-setup/main.tf b/community/modules/scripts/vdi-setup/main.tf index 83d6963eed..4ddb5a5215 100644 --- a/community/modules/scripts/vdi-setup/main.tf +++ b/community/modules/scripts/vdi-setup/main.tf @@ -39,6 +39,7 @@ locals { vdi_user_group = var.vdi_user_group vdi_webapp_port = var.vdi_webapp_port vdi_resolution = var.vdi_resolution + vdi_resolution_locked = var.vdi_resolution_locked vdi_users = var.vdi_users debug = var.debug reset_webapp_admin_password = var.reset_webapp_admin_password @@ -99,7 +100,7 @@ locals { } ) destination = "/opt/vdi-setup/install.yaml" - args = var.debug ? "--extra-vars @/opt/vdi-setup/vars.yaml -v --extra-vars debug=true" : "--extra-vars @/opt/vdi-setup/vars.yaml" + args = var.debug ? "--extra-vars @/opt/vdi-setup/vars.yaml -v --extra-vars debug=true" : "--extra-vars @/opt/vdi-setup/vars.yaml" }, # Clean up temporary directory { diff --git a/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml index 1a68dc133c..232a33abc5 100644 --- a/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/base_os/tasks/main.yaml @@ -26,15 +26,30 @@ ansible.builtin.import_role: name: lock_manager tasks_from: check_lock - register: lock_check_result - name: DEBUG Finished import_role for base_os debug: msg: "Finished import_role for base_os" -- name: Set role_should_run from lock check result +- name: Set role variables from lock manager facts ansible.builtin.set_fact: - role_should_run: "{{ lock_check_result.ansible_facts.role_should_run | default(true) }}" + role_should_run: "{{ role_should_run | default(false) }}" + is_fresh_deployment: "{{ is_fresh_deployment | default(false) }}" + deployment_hash: "{{ current_deployment_hash | default('none') }}" + user_hash: "{{ current_user_secrets_hash | default('none') }}" + lock_file_stat: "{{ lock_file_stat | default({}) }}" + vdi_setup_status: "{{ vdi_setup_status | default({}) }}" + +- name: Debug role variables + ansible.builtin.debug: + msg: | + Base OS role variables: + - role_should_run: {{ role_should_run }} + - is_fresh_deployment: {{ is_fresh_deployment }} + - deployment_hash: {{ deployment_hash }} + - user_hash: {{ user_hash }} + - execution_reason: {{ 'Fresh deployment' if is_fresh_deployment else ('Hash mismatch' if not role_should_run else 'Normal role execution') }} + when: debug # Skip all tasks if role should not run - name: Skip base_os tasks if role should not run @@ -42,60 +57,54 @@ msg: "Skipping base_os role - already completed or not needed" when: not role_should_run -- name: Gather distribution facts - ansible.builtin.setup: - gather_subset: platform - when: role_should_run - -# GPU / vWS driver setup -- name: Check if NVIDIA GPU is present - ansible.builtin.shell: lspci | grep -i nvidia - register: nvidia_gpu_present - failed_when: false - when: role_should_run +# Run base_os tasks only if needed +- name: Run base_os tasks + block: + + - name: Gather distribution facts + ansible.builtin.setup: + gather_subset: platform + + # GPU / vWS driver setup + - name: Check if NVIDIA GPU is present + ansible.builtin.shell: lspci | grep -i nvidia + register: nvidia_gpu_present + failed_when: false + + # Include distribution-specific tasks + - name: Include Ubuntu-specific tasks + ansible.builtin.include_tasks: ubuntu.yaml + when: ansible_facts.distribution == 'Ubuntu' + + - name: Include Rocky Linux specific tasks + ansible.builtin.include_tasks: rocky.yaml + when: ansible_facts.distribution == 'Rocky' + + - name: Include Debian-specific tasks + ansible.builtin.include_tasks: debian.yaml + when: ansible_facts.distribution == 'Debian' + + # Common tasks for all distributions + - name: Ensure requests package is installed + pip: + name: requests + state: present + + - name: Ensure Python & tooling are installed + ansible.builtin.package: + name: + - python3 + - python3-pip + - jq + state: present + + # GPU / vWS driver setup + - name: Provision GPU setup + ansible.builtin.include_tasks: gpu_driver_setup.yaml + when: nvidia_gpu_present.rc == 0 -# Include distribution-specific tasks -- name: Include Ubuntu-specific tasks - ansible.builtin.include_tasks: ubuntu.yaml - when: - - ansible_facts.distribution == 'Ubuntu' - - role_should_run - -- name: Include Rocky Linux specific tasks - ansible.builtin.include_tasks: rocky.yaml - when: - - ansible_facts.distribution == 'Rocky' - - role_should_run - -- name: Include Debian-specific tasks - ansible.builtin.include_tasks: debian.yaml - when: - - ansible_facts.distribution == 'Debian' - - role_should_run - -# Common tasks for all distributions -- name: Ensure requests package is installed - pip: - name: requests - state: present when: role_should_run -- name: Ensure Python & tooling are installed - ansible.builtin.package: - name: - - python3 - - python3-pip - - jq - state: present - when: role_should_run - -# GPU / vWS driver setup -- name: Provision GPU setup - ansible.builtin.include_tasks: gpu_driver_setup.yaml - when: - - nvidia_gpu_present.rc == 0 - - role_should_run - # Mark role as completed - name: Set current role for lock manager completion ansible.builtin.set_fact: diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/README.md b/community/modules/scripts/vdi-setup/roles/lock_manager/README.md new file mode 100644 index 0000000000..3111b2572c --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/lock_manager/README.md @@ -0,0 +1,150 @@ +# Lock Manager Role + +The `lock_manager` role is the central coordination mechanism for the VDI setup module. It manages deployment state, change detection, and ensures idempotent role execution. + +## Overview + +The lock manager uses a hash-based change detection system to determine when roles should run, preventing unnecessary re-execution while ensuring changes are properly applied. + +## Key Features + +### 1. **Hash-Based Change Detection** + +- **Deployment Hash**: Tracks changes to deployment configuration (ports, tools, etc.) +- **User Hash**: Tracks changes to user configurations (additions, removals, password resets) +- **Smart Detection**: Only re-runs roles when relevant changes are detected + +### 2. **Centralized Variable Management** + +- **User Management Variables**: Calculates and propagates user change information to all roles +- **Port Change Detection**: Detects and tracks VNC port changes for existing users +- **Password Reset Tracking**: Identifies users requiring password resets + +### 3. **Targeted Updates** + +- **Database-Only Updates**: Performs user additions/deletions/password resets without container recreation +- **Port Change Handling**: Updates VNC ports and Guacamole connections without full re-initialization +- **Webapp Admin Password Reset**: Updates admin password without container recreation + +### 4. **Deployment Status Management** + +- **Lock File**: `.vdi-lock.yaml` tracks deployment state and completed roles +- **Status Tracking**: `available`, `configuring`, `reconfiguring`, `failed` states +- **Role Completion**: Tracks which roles have completed successfully + +## Variable Propagation + +### **Variables Calculated by Lock Manager:** + +| Variable | Description | Usage | +|----------|-------------|-------| +| `new_users` | Users added since last deployment | `vdi_tool`, `user_provision`, `vnc` | +| `removed_users` | Users removed since last deployment | `vdi_tool`, `user_provision`, `vnc` | +| `users_needing_reset` | Users with `reset_password: true` | `vdi_tool`, `secret_manager` | +| `users_with_port_changes` | Users whose VNC ports changed | `vdi_tool`, `vnc` | +| `webapp_port_changed` | Whether webapp port changed | `vdi_tool` | +| `users_changed` | Any user-related changes detected | `vdi_tool` | +| `db_needs_init` | Whether database needs re-initialization | `vdi_tool` | +| `vdi_user_group` | Current VDI user group for group management | `user_provision` | + +### **Variable Propagation Flow:** + +1. **Lock Manager Calculation**: Variables calculated once during initial `lock_manager` run +2. **Role Import**: Other roles import `lock_manager` and receive these variables +3. **Defensive Programming**: Variables preserved with `default()` filters, not overwritten +4. **Role Usage**: Each role uses only the variables it needs + +## Data Flows + +### **Initial Deployment:** + +```mermaid +1. lock_manager ā†’ Calculates all variables (empty for fresh deployment) +2. base_os ā†’ OS setup +3. secret_manager ā†’ Password generation and Secret Manager setup +4. user_provision ā†’ Local user creation +5. vnc ā†’ VNC service setup +6. vdi_tool ā†’ Guacamole deployment +7. vdi_monitor ā†’ Monitoring service setup +``` + +### **Reconfiguration (User Changes):** + +```mermaid +1. lock_manager ā†’ Detects changes, calculates user management variables +2. user_provision ā†’ Handles user additions/removals +3. vnc ā†’ Handles VNC port changes and user removal +4. vdi_tool ā†’ Handles database updates (targeted, no container recreation) +5. vdi_monitor ā†’ No changes needed +``` + +### **Reconfiguration (Port Changes):** + +```mermaid +1. lock_manager ā†’ Detects port changes +2. vnc ā†’ Stops VNC services for users with port changes +3. vdi_tool ā†’ Updates Guacamole connection ports in database +4. user_provision ā†’ Updates local user configurations +``` + +## Lock File Structure + +```yaml +vdi_setup_status: + completed_roles: + base_os: true + secret_manager: true + user_provision: true + vdi_monitor: true + vdi_tool: true + vnc: true + created_at: '2025-08-07T16:20:26Z' + current_users: ['alice', 'bob', 'charlie', 'david'] + deployment_hash: 'abc123...' + deployment_name: 'vdi-test-scott' + force_rerun: false + last_updated: '2025-08-07T16:28:42Z' + lock_version: '1.0' + setup_status: 'available' + user_hash: 'def456...' + user_ports: + alice: 5901 + bob: 5902 + charlie: 5903 + david: 5904 + webapp_port: 8081 + vdi_user_group: vdiusers +``` + +## Role Execution Logic + +### **When Roles Run:** +- **Fresh Deployment**: All roles run regardless of hash +- **Hash Mismatch**: Only roles with changed hashes run +- **Force Rerun**: All roles run when `force_rerun: true` + +### **Role Completion:** +- Roles mark themselves as completed in lock file +- Lock file tracks completion status per role +- Deployment status updated based on all role completions + +## Error Handling + +### **Failed Deployments:** +- VDI monitor sets status to `"failed"` when Ansible exits with non-zero code +- Lock file preserves failed state for debugging +- Valid blueprint can be re-applied to fix `failed` deployments + +### **Variable Safety:** +- Defensive programming ensures all variables have safe defaults +- `default()` filters prevent undefined variable errors +- Variables preserved across role imports + +## Debug Output + +When `debug: true` is set, the lock manager provides detailed information about: +- Variable calculations and propagation +- Change detection results +- Hash comparisons +- Role execution decisions +- Port change detection details diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/check_lock.yaml b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/check_lock.yaml index 86807a9470..35578df559 100644 --- a/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/check_lock.yaml +++ b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/check_lock.yaml @@ -21,7 +21,7 @@ - name: Debug lock file existence ansible.builtin.debug: msg: "Lock file exists: {{ lock_file_stat.stat.exists }}" - when: debug | default(false) + when: debug - name: Load existing lock file ansible.builtin.include_vars: @@ -29,22 +29,144 @@ when: lock_file_stat.stat.exists register: lock_file_load +- name: Set vdi_setup_status from loaded lock file + ansible.builtin.set_fact: + vdi_setup_status: "{{ vdi_setup_status | default({}) }}" + when: lock_file_stat.stat.exists + +- name: Set vdi_setup_status to empty dict for fresh deployments + ansible.builtin.set_fact: + vdi_setup_status: {} + when: not lock_file_stat.stat.exists + +- name: Debug vdi_setup_status after loading + ansible.builtin.debug: + msg: | + vdi_setup_status after loading: + - vdi_setup_status is defined: {{ vdi_setup_status is defined }} + - vdi_setup_status type: {{ vdi_setup_status | type_debug }} + - deployment_hash: {{ vdi_setup_status.deployment_hash | default('not_found') }} + - keys: {{ vdi_setup_status.keys() | default([]) }} + when: debug and lock_file_stat.stat.exists + - name: Debug lock file load result ansible.builtin.debug: msg: "Lock file loaded successfully: {{ lock_file_load is defined }}" - when: debug | default(false) and lock_file_stat.stat.exists + when: debug and lock_file_stat.stat.exists + +# ============================================================================= +# DEFENSIVE PROGRAMMING: SET DEFAULTS FOR ALL CRITICAL VARIABLES +# ============================================================================= +# This ensures that debug tasks and other logic never fail due to undefined variables +# All variables that might be referenced in debug output or conditional logic are set here +# Set early so we can remove default() filters throughout the rest of the file + +- name: Set default values for critical variables (defensive programming) + ansible.builtin.set_fact: + is_fresh_deployment: "{{ is_fresh_deployment | default(not lock_file_stat.stat.exists) }}" + db_needs_init: "{{ db_needs_init | default(false) }}" + users_needing_reset: "{{ users_needing_reset | default([]) }}" + force_rerun: "{{ force_rerun | default(false) }}" + reset_webapp_admin_password: "{{ reset_webapp_admin_password | default(false) }}" + debug: "{{ debug | default(false) }}" + # Initialize deployment variables with safe defaults + deployment_name: "{{ deployment_name | default('vdi-deployment') }}" + project_id: "{{ project_id | default('unknown-project') }}" + vdi_tool: "{{ vdi_tool | default('guacamole') }}" + vnc_flavor: "{{ vnc_flavor | default('tigervnc') }}" + user_provision: "{{ user_provision | default('local_users') }}" + vdi_webapp_port: "{{ vdi_webapp_port | default(8080) }}" + vdi_user_group: "{{ vdi_user_group | default('vdiusers') }}" + vdi_resolution: "{{ vdi_resolution | default('1920x1080') }}" + vdi_resolution_locked: "{{ vdi_resolution_locked | default(true) }}" + vdi_users: "{{ vdi_users | default([]) }}" + # Initialize user management variables with safe defaults (only if not already set) + current_users: "{{ current_users | default([]) }}" + previous_users: "{{ previous_users | default([]) }}" + new_user_names: "{{ new_user_names | default([]) }}" + removed_users: "{{ removed_users | default([]) }}" + users_changed: "{{ users_changed | default(false) }}" + new_users: "{{ new_users | default([]) }}" + removed_user_objects: "{{ removed_user_objects | default([]) }}" + # Initialize port change detection variables with safe defaults (only if not already set) + users_with_port_changes: "{{ users_with_port_changes | default([]) }}" + webapp_port_changed: "{{ webapp_port_changed | default(false) }}" + vdi_resolution_changed: "{{ vdi_resolution_changed | default(false) }}" + vdi_resolution_locked_changed: "{{ vdi_resolution_locked_changed | default(false) }}" + infrastructure_changed: "{{ infrastructure_changed | default(false) }}" + # Initialize secret manager variables with safe defaults + webapp_admin_hash: "{{ webapp_admin_hash | default('') }}" + +- name: Calculate user management variables (when vdi_users is available) + ansible.builtin.set_fact: + current_users: "{{ vdi_users | map(attribute='username') | list }}" + previous_users: "{{ vdi_setup_status.current_users | default([]) }}" + new_user_names: "{{ current_users | difference(previous_users) }}" + removed_users: "{{ previous_users | difference(current_users) }}" + users_changed: "{{ (new_user_names | length > 0) or (removed_users | length > 0) }}" + new_users: "{{ vdi_users | selectattr('username', 'in', new_user_names) | list if new_user_names | length > 0 else [] }}" + removed_user_objects: "{{ vdi_users | selectattr('username', 'in', removed_users) | list if removed_users | length > 0 else [] }}" + users_needing_reset: "{{ vdi_users | selectattr('reset_password', 'defined') | selectattr('reset_password', 'equalto', true) | list }}" + when: vdi_users is defined + +- name: Detect port changes + ansible.builtin.set_fact: + users_with_port_changes: "{{ users_with_port_changes | default([]) }}" + webapp_port_changed: "{{ vdi_webapp_port != vdi_setup_status.webapp_port | default(false) }}" + vdi_resolution_changed: "{{ vdi_resolution != vdi_setup_status.vdi_resolution | default(false) }}" + vdi_resolution_locked_changed: "{{ vdi_resolution_locked != vdi_setup_status.vdi_resolution_locked | default(false) }}" + infrastructure_changed: "{{ webapp_port_changed or vdi_resolution_changed or vdi_resolution_locked_changed or reset_webapp_admin_password }}" + when: vdi_users is defined and not is_fresh_deployment + +- name: Calculate user port changes (only if not already calculated) + ansible.builtin.set_fact: + users_with_port_changes: "{{ users_with_port_changes_temp | default([]) }}" + when: vdi_users is defined and not is_fresh_deployment and (users_with_port_changes | default([]) | length == 0) + +- name: Calculate port changes for each user + ansible.builtin.set_fact: + users_with_port_changes_temp: "{{ users_with_port_changes_temp | default([]) + [item] }}" + loop: "{{ vdi_users | selectattr('username', 'in', vdi_setup_status.current_users | default([])) | selectattr('port', 'defined') | list }}" + when: > + vdi_users is defined and + not is_fresh_deployment and + (users_with_port_changes | default([]) | length == 0) and + vdi_setup_status.user_ports[item.username] is defined and + item.port != vdi_setup_status.user_ports[item.username] + +- name: Debug port change detection + ansible.builtin.debug: + msg: | + Port change detection results: + - users_with_port_changes: {{ users_with_port_changes | default([]) }} + - vdi_setup_status.user_ports: {{ vdi_setup_status.user_ports | default({}) }} + - webapp_port_changed: {{ webapp_port_changed }} + - current_webapp_port: {{ vdi_webapp_port }} + - previous_webapp_port: {{ vdi_setup_status.webapp_port | default('none') }} + - vdi_resolution_changed: {{ vdi_resolution_changed }} + - current_vdi_resolution: {{ vdi_resolution }} + - previous_vdi_resolution: {{ vdi_setup_status.vdi_resolution | default('none') }} + - vdi_resolution_locked_changed: {{ vdi_resolution_locked_changed }} + - current_vdi_resolution_locked: {{ vdi_resolution_locked }} + - previous_vdi_resolution_locked: {{ vdi_setup_status.vdi_resolution_locked | default('none') }} + - infrastructure_changed: {{ infrastructure_changed }} + when: vdi_users is defined and not is_fresh_deployment and debug - name: Calculate current deployment hash ansible.builtin.set_fact: current_deployment_hash: >- {{ - (deployment_name | string + - project_id | string + - vdi_tool | string + - vnc_flavor | string + - user_provision | string + - vdi_webapp_port | string + - vdi_resolution | string + + (deployment_name | string + + project_id | string + + vdi_tool | string + + vnc_flavor | string + + user_provision | string + + vdi_webapp_port | string + + vdi_resolution | string + + vdi_resolution_locked | string + + vdi_user_group | string + + (reset_webapp_admin_password | string) + + (force_rerun | string) + (vdi_users | to_json)) | hash('sha256') }} @@ -60,8 +182,12 @@ - user_provision: {{ user_provision }} - vdi_webapp_port: {{ vdi_webapp_port }} - vdi_resolution: {{ vdi_resolution }} + - vdi_resolution_locked: {{ vdi_resolution_locked }} + - vdi_user_group: {{ vdi_user_group }} + - reset_webapp_admin_password: {{ reset_webapp_admin_password }} + - force_rerun: {{ force_rerun }} - vdi_users count: {{ vdi_users | length }} - when: debug | default(false) + when: debug - name: Calculate current user secrets hash ansible.builtin.set_fact: @@ -73,50 +199,24 @@ Current user secrets hash: {{ current_user_secrets_hash }} User configuration components: - vdi_users (blueprint): {{ vdi_users | to_json }} - when: debug | default(false) - -- name: Set default values for missing lock file - ansible.builtin.set_fact: - vdi_setup_status: - deployment_name: "{{ deployment_name }}" - deployment_hash: "none" - lock_version: "1.0" - created_at: "{{ ansible_date_time.iso8601 }}" - last_updated: "{{ ansible_date_time.iso8601 }}" - force_rerun: false - setup_status: "configuring" - completed_roles: - base_os: - completed: false - lock_manager: - completed: false - secret_manager: - completed: false - user_provision: - completed: false - vnc: - completed: false - vdi_tool: - completed: false - user_secrets_status: - last_secret_check: "{{ ansible_date_time.iso8601 }}" - user_secrets_hash: "none" - users_updated: [] - user_management: - current_users: [] - previous_users: [] - removed_users: [] - when: not lock_file_stat.stat.exists + when: debug + +- name: Store original lock file state for this deployment run + ansible.builtin.set_fact: + original_lock_file_exists: "{{ lock_file_stat.stat.exists }}" + when: original_lock_file_exists is not defined - name: Debug lock file status ansible.builtin.debug: msg: | Lock file status: - - exists: {{ lock_file_stat.stat.exists }} - - deployment_hash: {{ vdi_setup_status.deployment_hash }} - - user_secrets_hash: {{ vdi_setup_status.user_secrets_status.user_secrets_hash }} - - force_rerun: {{ vdi_setup_status.force_rerun }} - when: debug | default(false) + - current_exists: {{ lock_file_stat.stat.exists }} + - original_exists: {{ original_lock_file_exists | default('not_set') }} + - deployment_hash: {{ vdi_setup_status.deployment_hash | default('none') }} + - user_secrets_hash: {{ vdi_setup_status.user_secrets_status.user_secrets_hash | default('none') }} + - force_rerun: {{ vdi_setup_status.force_rerun | default(false) }} + - is_fresh_deployment: {{ is_fresh_deployment }} + when: debug - name: Check if lock file exists ansible.builtin.set_fact: @@ -124,30 +224,35 @@ - name: Check if current role is completed ansible.builtin.set_fact: - current_role_completed: "{{ vdi_setup_status.completed_roles[current_role | default(ansible_role_name | basename)].completed | default(false) }}" + current_role_completed: "{{ vdi_setup_status.completed_roles[current_role | default(ansible_role_name | basename)] | default(false) if lock_file_stat.stat.exists else false }}" - name: Check if deployment hash matches ansible.builtin.set_fact: - deployment_hash_matches: "{{ vdi_setup_status.deployment_hash == current_deployment_hash }}" + deployment_hash_matches: "{{ (vdi_setup_status.deployment_hash | default('none')) == current_deployment_hash if lock_file_stat.stat.exists and vdi_setup_status is defined else false }}" + +- name: Check if user hash matches + ansible.builtin.set_fact: + user_hash_matches: "{{ (vdi_setup_status.user_hash | default('none')) == current_user_secrets_hash if lock_file_stat.stat.exists and vdi_setup_status is defined else false }}" - name: Check if force rerun is enabled ansible.builtin.set_fact: - force_rerun_enabled: "{{ vdi_setup_status.force_rerun | default(false) }}" + force_rerun_enabled: "{{ force_rerun | default(false) }}" -- name: Determine if role should run +- name: Determine if role should run (hash-based) ansible.builtin.set_fact: - role_should_run: "{{ not lock_file_exists or not current_role_completed or not deployment_hash_matches or force_rerun_enabled }}" + role_should_run: "{{ is_fresh_deployment or not current_role_completed or not deployment_hash_matches or not user_hash_matches or force_rerun_enabled }}" - name: Debug role should run calculation ansible.builtin.debug: msg: | Role should run calculation for {{ current_role | default(ansible_role_name | basename) }}: - - not lock_file_exists: {{ not lock_file_exists }} + - is_fresh_deployment: {{ is_fresh_deployment }} - not current_role_completed: {{ not current_role_completed }} - not deployment_hash_matches: {{ not deployment_hash_matches }} + - not user_hash_matches: {{ not user_hash_matches }} - force_rerun_enabled: {{ force_rerun_enabled }} - - role_should_run: {{ not lock_file_exists or not current_role_completed or not deployment_hash_matches or force_rerun_enabled }} - when: debug | default(false) + - role_should_run: {{ role_should_run }} + when: debug - name: Debug role execution decision ansible.builtin.debug: @@ -156,48 +261,51 @@ - current_role: {{ current_role | default('not set') }} - ansible_role_name: {{ ansible_role_name }} - role_should_run: {{ role_should_run }} - - lock_file_exists: {{ lock_file_exists }} + - is_fresh_deployment: {{ is_fresh_deployment }} - current_role_completed: {{ current_role_completed }} - deployment_hash_matches: {{ deployment_hash_matches }} + - user_hash_matches: {{ user_hash_matches }} - force_rerun_enabled: {{ force_rerun_enabled }} - when: debug | default(false) + when: debug -- name: Check if user configuration has changed - ansible.builtin.set_fact: - user_config_changed: "{{ not lock_file_stat.stat.exists or vdi_setup_status.user_secrets_status.user_secrets_hash != current_user_secrets_hash }}" - -- name: Set user secrets changed flag (will be updated by secret_manager role) - ansible.builtin.set_fact: - user_secrets_changed: "{{ user_config_changed }}" - -- name: Debug user secrets change detection +- name: Debug lock manager completion ansible.builtin.debug: - msg: | - User configuration change detection: - - user_config_changed: {{ user_config_changed }} - - user_secrets_changed: {{ user_secrets_changed }} - - stored_hash: {{ vdi_setup_status.user_secrets_status.user_secrets_hash }} - - current_hash: {{ current_user_secrets_hash }} - when: debug | default(false) - -- name: Set lock check result - ansible.builtin.set_fact: - lock_check_result: - role_should_run: "{{ role_should_run }}" - current_deployment_hash: "{{ current_deployment_hash }}" - existing_deployment_hash: "{{ vdi_setup_status.deployment_hash | default('none') }}" - user_secrets_changed: "{{ user_secrets_changed }}" - current_user_secrets_hash: "{{ current_user_secrets_hash }}" - existing_user_secrets_hash: "{{ vdi_setup_status.user_secrets_status.user_secrets_hash | default('none') }}" - force_rerun: "{{ vdi_setup_status.force_rerun | default(false) }}" - -- name: Debug final lock check result + msg: "Lock manager role completed successfully" + +- name: Summary of changes and lock file contents ansible.builtin.debug: msg: | - Final lock check result: - - role_should_run: {{ lock_check_result.role_should_run }} - - deployment_hash_changed: {{ lock_check_result.current_deployment_hash != lock_check_result.existing_deployment_hash }} - - user_secrets_changed: {{ lock_check_result.user_secrets_changed }} - - force_rerun: {{ lock_check_result.force_rerun }} - when: debug | default(false) - \ No newline at end of file + ================================================================================ + VDI DEPLOYMENT SUMMARY + ================================================================================ + + DEPLOYMENT STATUS: + - is_fresh_deployment: {{ is_fresh_deployment }} + - deployment_hash: {{ current_deployment_hash }} + - user_hash: {{ current_user_secrets_hash }} + - reset_webapp_admin_password: {{ reset_webapp_admin_password }} + + USER CHANGES DETECTED: + - users_changed: {{ users_changed }} + - new_users: {{ new_users | map(attribute='username') | list if new_users is defined and new_users | length > 0 else [] }} + - removed_users: {{ removed_users if removed_users is defined and removed_users | length > 0 else [] }} + - users_needing_reset: {{ users_needing_reset | map(attribute='username') | list if users_needing_reset is defined and users_needing_reset | length > 0 else [] }} + - users_with_port_changes: {{ users_with_port_changes | default([]) }} + - webapp_port_changed: {{ webapp_port_changed }} + - vdi_resolution_changed: {{ vdi_resolution_changed }} + - vdi_resolution_locked_changed: {{ vdi_resolution_locked_changed }} + + CURRENT LOCK FILE CONTENTS: + - completed_roles: {{ vdi_setup_status.completed_roles | default({}) }} + - current_users: {{ vdi_setup_status.current_users | default([]) }} + - user_ports: {{ vdi_setup_status.user_ports | default({}) }} + - webapp_port: {{ vdi_setup_status.webapp_port | default('none') }} + - vdi_resolution: {{ vdi_setup_status.vdi_resolution | default('none') }} + - vdi_resolution_locked: {{ vdi_setup_status.vdi_resolution_locked | default('none') }} + - deployment_hash: {{ vdi_setup_status.deployment_hash | default('none') }} + - user_hash: {{ vdi_setup_status.user_hash | default('none') }} + - setup_status: {{ vdi_setup_status.setup_status | default('unknown') }} + - last_updated: {{ vdi_setup_status.last_updated | default('unknown') }} + + ================================================================================ + when: debug and current_role == 'lock_manager' diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/cleanup_lock.yaml b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/cleanup_lock.yaml index c6b1eec63a..e90f2bdef6 100644 --- a/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/cleanup_lock.yaml +++ b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/cleanup_lock.yaml @@ -26,4 +26,4 @@ - name: Debug lock file removal ansible.builtin.debug: msg: "Lock file removed: {{ lock_file_removal.changed }}" - when: debug | default(false) + when: debug diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/create_lock.yaml b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/create_lock.yaml index 1d1bb1f61b..411cd37f5e 100644 --- a/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/create_lock.yaml +++ b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/create_lock.yaml @@ -13,6 +13,14 @@ # limitations under the License. --- +- name: Backup existing lock file + ansible.builtin.copy: + src: /opt/vdi-setup/.vdi-lock.yaml + dest: /opt/vdi-setup/.vdi-lock.yaml.backup + remote_src: true + when: lock_file_stat.stat.exists + ignore_errors: true + - name: Debug create lock operation ansible.builtin.debug: msg: "Creating/updating lock file for role: {{ current_role }}" @@ -25,7 +33,7 @@ - name: Debug lock file existence ansible.builtin.debug: msg: "Lock file exists: {{ lock_file_stat.stat.exists }}" - when: debug | default(false) + when: debug - name: Create initial lock file structure ansible.builtin.template: @@ -38,7 +46,7 @@ - name: Debug lock file creation ansible.builtin.debug: msg: "Lock file created: {{ lock_file_creation.changed }}" - when: debug | default(false) and not lock_file_stat.stat.exists + when: debug and not lock_file_stat.stat.exists - name: Set fresh deployment flag (only when lock file is created) ansible.builtin.set_fact: @@ -56,76 +64,56 @@ current_lock_data: "{{ existing_lock_data.ansible_facts.vdi_setup_status | default({}) }}" when: lock_file_stat.stat.exists or lock_file_creation.changed -- name: Update deployment hash +# Update hashes and configuration when lock manager runs independently OR when secret_manager provides admin hash +- name: Update deployment hash (independent run only) ansible.builtin.set_fact: current_lock_data: "{{ current_lock_data | combine({'deployment_hash': current_deployment_hash}) }}" - when: lock_file_stat.stat.exists or lock_file_creation.changed - -- name: Get current timestamp - ansible.builtin.command: date -u +%Y-%m-%dT%H:%M:%SZ - register: current_timestamp - when: lock_file_stat.stat.exists or lock_file_creation.changed - -- name: Update last updated timestamp - ansible.builtin.set_fact: - current_lock_data: "{{ current_lock_data | combine({'last_updated': current_timestamp.stdout}) }}" - when: lock_file_stat.stat.exists or lock_file_creation.changed - -- name: Get current user secrets status - ansible.builtin.set_fact: - current_user_secrets: "{{ current_lock_data.user_secrets_status | default({}) }}" - when: lock_file_stat.stat.exists or lock_file_creation.changed - -- name: Create user secrets update - ansible.builtin.set_fact: - user_secrets_update: - user_secrets_hash: "{{ current_user_secrets_hash }}" - last_secret_check: "{{ current_timestamp.stdout }}" - when: lock_file_stat.stat.exists or lock_file_creation.changed - -- name: Update user secrets hash - ansible.builtin.set_fact: - updated_user_secrets: "{{ current_user_secrets | combine(user_secrets_update) }}" - when: lock_file_stat.stat.exists or lock_file_creation.changed - -- name: Apply user secrets update - ansible.builtin.set_fact: - current_lock_data: "{{ current_lock_data | combine({'user_secrets_status': updated_user_secrets}) }}" - when: lock_file_stat.stat.exists or lock_file_creation.changed - -- name: Create role status entry - ansible.builtin.set_fact: - role_status_entry: - completed: "{{ role_completed | default(false) }}" - completed_at: "{{ current_timestamp.stdout }}" - when: lock_file_stat.stat.exists or lock_file_creation.changed - -- name: Debug role status entry - ansible.builtin.debug: - msg: | - Role status entry for {{ current_role | default(ansible_role_name | basename) }}: - - completed: {{ role_completed | default(false) }} - - completed_at: {{ current_timestamp.stdout }} - when: debug | default(false) and (lock_file_stat.stat.exists or lock_file_creation.changed) - -- name: Debug current lock data before role update - ansible.builtin.debug: - msg: | - Current lock data before updating {{ current_role }}: - - current_lock_data: {{ current_lock_data | to_nice_yaml }} - - current_completed_roles: {{ current_lock_data.completed_roles | default({}) | to_nice_yaml }} - - role_completed: {{ role_completed }} - - current_role: {{ current_role }} - when: debug | default(false) and (lock_file_stat.stat.exists or lock_file_creation.changed) and role_completed is defined - -- name: Get current completed roles + when: + - lock_file_stat.stat.exists or lock_file_creation.changed + - current_role == 'lock_manager' + +- name: Update user hash (independent run only) + ansible.builtin.set_fact: + current_lock_data: "{{ current_lock_data | combine({'user_hash': current_user_secrets_hash}) }}" + when: + - lock_file_stat.stat.exists or lock_file_creation.changed + - current_role == 'lock_manager' + +- name: Prepare configuration updates + ansible.builtin.set_fact: + config_updates: + current_users: "{{ vdi_users | map(attribute='username') | list }}" + user_ports: "{{ vdi_users | items2dict(key_name='username', value_name='port') }}" + webapp_port: "{{ vdi_webapp_port }}" + vdi_resolution: "{{ vdi_resolution }}" + vdi_resolution_locked: "{{ vdi_resolution_locked }}" + vdi_user_group: "{{ vdi_user_group }}" + webapp_admin_hash: "{{ webapp_admin_hash }}" + last_updated: "{{ ansible_date_time.iso8601 }}" + when: + - lock_file_stat.stat.exists or lock_file_creation.changed + - current_role == 'lock_manager' + +- name: Update configuration data (independent run only) + ansible.builtin.set_fact: + current_lock_data: "{{ current_lock_data | combine(config_updates) }}" + when: + - lock_file_stat.stat.exists or lock_file_creation.changed + - current_role == 'lock_manager' + +# Update admin hash when secret_manager role provides it +- name: Update admin hash from secret_manager + ansible.builtin.set_fact: + current_lock_data: "{{ current_lock_data | combine({'webapp_admin_hash': webapp_admin_hash}) }}" + when: + - lock_file_stat.stat.exists or lock_file_creation.changed + - current_role == 'secret_manager' + - webapp_admin_hash is defined + +- name: Update role completion status ansible.builtin.set_fact: current_completed_roles: "{{ current_lock_data.completed_roles | default({}) }}" - when: lock_file_stat.stat.exists or lock_file_creation.changed - -- name: Update completed roles - ansible.builtin.set_fact: - updated_completed_roles: "{{ current_completed_roles | combine({current_role | default(ansible_role_name | basename): role_status_entry}) }}" + updated_completed_roles: "{{ (current_lock_data.completed_roles | default({})) | combine({current_role: role_completed | default(false)}) }}" when: lock_file_stat.stat.exists or lock_file_creation.changed - name: Update lock data with completed roles @@ -133,106 +121,6 @@ current_lock_data: "{{ current_lock_data | combine({'completed_roles': updated_completed_roles}) }}" when: lock_file_stat.stat.exists or lock_file_creation.changed -- name: Debug updated lock data after role update - ansible.builtin.debug: - msg: | - Updated lock data after updating {{ current_role | default(ansible_role_name | basename) }}: - - current_role: {{ current_role | default(ansible_role_name | basename) }} - - role_completed: {{ role_completed | default(false) }} - - completed_roles: {{ updated_completed_roles }} - when: debug | default(false) and (lock_file_stat.stat.exists or lock_file_creation.changed) - -- name: Get current users updated list - ansible.builtin.set_fact: - current_users_updated: "{{ current_lock_data.user_secrets_status.users_updated | default([]) }}" - when: (lock_file_stat.stat.exists or lock_file_creation.changed) and user_secrets_changed - -- name: Get new usernames list - ansible.builtin.set_fact: - new_usernames: "{{ vdi_users | map(attribute='username') | list }}" - when: (lock_file_stat.stat.exists or lock_file_creation.changed) and user_secrets_changed - -- name: Combine users updated lists - ansible.builtin.set_fact: - combined_users_updated: "{{ (current_users_updated + new_usernames) | unique | list }}" - when: (lock_file_stat.stat.exists or lock_file_creation.changed) and user_secrets_changed - -- name: Debug user deduplication - ansible.builtin.debug: - msg: | - User deduplication for {{ current_role }}: - - current_users_updated: {{ current_users_updated }} - - new_usernames: {{ new_usernames }} - - combined_users_updated: {{ combined_users_updated }} - when: debug | default(false) and (lock_file_stat.stat.exists or lock_file_creation.changed) and user_secrets_changed - -- name: Get current user secrets for list update - ansible.builtin.set_fact: - current_user_secrets_for_list: "{{ current_lock_data.user_secrets_status }}" - when: (lock_file_stat.stat.exists or lock_file_creation.changed) and user_secrets_changed - -- name: Update users updated list - ansible.builtin.set_fact: - updated_user_secrets_with_list: "{{ current_user_secrets_for_list | combine({'users_updated': combined_users_updated}) }}" - when: (lock_file_stat.stat.exists or lock_file_creation.changed) and user_secrets_changed - -- name: Apply users updated list - ansible.builtin.set_fact: - current_lock_data: "{{ current_lock_data | combine({'user_secrets_status': updated_user_secrets_with_list}) }}" - when: (lock_file_stat.stat.exists or lock_file_creation.changed) and user_secrets_changed - -- name: Get current user management data - ansible.builtin.set_fact: - current_user_management: "{{ current_lock_data.user_management | default({}) }}" - when: lock_file_stat.stat.exists or lock_file_creation.changed - -- name: Get current usernames list - ansible.builtin.set_fact: - current_usernames: "{{ vdi_users | map(attribute='username') | list }}" - when: lock_file_stat.stat.exists or lock_file_creation.changed - -- name: Update user management data - ansible.builtin.set_fact: - updated_user_management: - previous_users: "{{ current_user_management.current_users | default([]) }}" - current_users: "{{ current_usernames }}" - removed_users: "{{ (current_user_management.current_users | default([])) | difference(current_usernames) }}" - when: lock_file_stat.stat.exists or lock_file_creation.changed - -- name: Apply user management update - ansible.builtin.set_fact: - current_lock_data: "{{ current_lock_data | combine({'user_management': updated_user_management}) }}" - when: lock_file_stat.stat.exists or lock_file_creation.changed - -- name: Debug user management - ansible.builtin.debug: - msg: | - User management for {{ current_role }}: - - current_users: {{ current_usernames }} - - previous_users: {{ current_user_management.current_users | default([]) }} - - removed_users: {{ (current_user_management.current_users | default([])) | difference(current_usernames) }} - when: debug | default(false) and (lock_file_stat.stat.exists or lock_file_creation.changed) - -- name: Check if all roles are completed - ansible.builtin.set_fact: - all_roles_completed: "{{ updated_completed_roles.values() | selectattr('completed', 'equalto', true) | list | length == updated_completed_roles.keys() | list | length }}" - when: lock_file_stat.stat.exists or lock_file_creation.changed - -- name: Update setup status based on completion - ansible.builtin.set_fact: - current_lock_data: "{{ current_lock_data | combine({'setup_status': 'available' if all_roles_completed else 'configuring'}) }}" - when: lock_file_stat.stat.exists or lock_file_creation.changed - -- name: Debug setup status update - ansible.builtin.debug: - msg: | - Setup status update: - - all_roles_completed: {{ all_roles_completed | default(false) }} - - setup_status: {{ 'available' if all_roles_completed else 'configuring' }} - - completed_roles_count: {{ updated_completed_roles.values() | selectattr('completed', 'equalto', true) | list | length }} - - total_roles_count: {{ updated_completed_roles.keys() | list | length }} - when: debug | default(false) and (lock_file_stat.stat.exists or lock_file_creation.changed) - - name: Write updated lock file ansible.builtin.copy: content: | @@ -250,7 +138,7 @@ - hostname: {{ ansible_hostname }} - zone: {{ zone }} - lock_file_changed: {{ lock_file_write.changed }} - when: debug | default(false) + when: debug - name: Get lock file content as base64 ansible.builtin.shell: cat /opt/vdi-setup/.vdi-lock.yaml | base64 -w 0 @@ -262,22 +150,22 @@ gcloud compute instances add-metadata {{ ansible_hostname }} --metadata vdi-lock-content="{{ lock_file_base64.stdout }}" --zone={{ zone }} - when: - - lock_file_write.changed - - lock_file_base64.rc == 0 + when: + - lock_file_write.changed + - lock_file_base64.rc == 0 register: metadata_update retries: 3 delay: 2 until: metadata_update.rc == 0 changed_when: metadata_update.rc == 0 - failed_when: - - metadata_update.rc != 0 - - "'already exists' not in metadata_update.stderr | default('')" + failed_when: + - metadata_update.rc != 0 + - "'already exists' not in metadata_update.stderr | default('')" - name: Debug lock file write ansible.builtin.debug: msg: "Lock file written: {{ lock_file_write.changed }}" - when: debug | default(false) + when: debug - name: Verify lock file integrity ansible.builtin.command: python3 -c "import yaml; yaml.safe_load(open('/opt/vdi-setup/.vdi-lock.yaml'))" @@ -287,15 +175,14 @@ - name: Debug lock file validation ansible.builtin.debug: msg: "Lock file validation: {{ lock_file_validation.rc == 0 }}" - when: debug | default(false) + when: debug - name: Debug lock manager completion ansible.builtin.debug: msg: "Lock manager role completed successfully" - when: debug | default(false) + when: debug - name: Fail if lock file is invalid ansible.builtin.fail: msg: "Lock file validation failed: {{ lock_file_validation.stderr }}" when: lock_file_validation.rc != 0 - \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/main.yaml index 3d236a94d0..7802068833 100644 --- a/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/lock_manager/tasks/main.yaml @@ -31,7 +31,7 @@ - name: Debug lock directory status ansible.builtin.debug: msg: "Lock directory created: {{ lock_dir_creation.changed }}" - when: debug | default(false) + when: debug - name: Include check lock tasks ansible.builtin.include_tasks: check_lock.yaml @@ -48,14 +48,3 @@ - name: Debug lock manager role completion ansible.builtin.debug: msg: "Lock manager role completed - operation: {{ lock_operation | default('check') }}" - -# Mark lock_manager role as completed when running as a standalone role -- name: Mark lock_manager role as completed - ansible.builtin.include_tasks: create_lock.yaml - vars: - current_role: "lock_manager" - role_completed: true - when: - - current_role == "lock_manager" - - lock_operation | default('check') == 'check' - \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 b/community/modules/scripts/vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 index b168ccc866..eac4765c08 100644 --- a/community/modules/scripts/vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 +++ b/community/modules/scripts/vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 @@ -4,35 +4,25 @@ vdi_setup_status: deployment_name: "{{ deployment_name }}" deployment_hash: "{{ current_deployment_hash }}" + user_hash: "{{ current_user_secrets_hash }}" + current_users: {{ vdi_users | map(attribute='username') | list | to_json }} + user_ports: {{ vdi_users | items2dict(key_name='username', value_name='port') | to_json }} + webapp_port: {{ vdi_webapp_port }} + vdi_resolution: {{ vdi_resolution | default('1920x1080') }} + vdi_resolution_locked: {{ vdi_resolution_locked | default(true) }} + webapp_admin_hash: "{{ webapp_admin_hash | default('') }}" + vdi_user_group: "{{ vdi_user_group | default('vdiusers') }}" lock_version: "1.0" created_at: "{{ ansible_date_time.iso8601 }}" last_updated: "{{ ansible_date_time.iso8601 }}" - force_rerun: false + force_rerun: {{ force_rerun | default(false) }} setup_status: "configuring" # configuring, available, error - + completed_roles: - lock_manager: - completed: false - base_os: - completed: false - secret_manager: - completed: false - user_provision: - completed: false - vnc: - completed: false - vdi_tool: - completed: false - vdi_monitor: - completed: false - - user_secrets_status: - last_secret_check: "{{ ansible_date_time.iso8601 }}" - user_secrets_hash: "{{ current_user_secrets_hash }}" - users_updated: [] - - user_management: - current_users: [] - previous_users: [] - removed_users: [] - \ No newline at end of file + base_os: false + secret_manager: false + user_provision: false + vnc: false + vdi_tool: false + vdi_monitor: false + diff --git a/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml index 6bbd23b9b3..c013d042c4 100644 --- a/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/main.yaml @@ -22,11 +22,26 @@ ansible.builtin.import_role: name: lock_manager tasks_from: check_lock - register: lock_check_result -- name: Set role_should_run from lock check result +- name: Set role variables from lock manager facts ansible.builtin.set_fact: - role_should_run: "{{ lock_check_result.ansible_facts.role_should_run | default(true) }}" + role_should_run: "{{ role_should_run | default(false) }}" + is_fresh_deployment: "{{ is_fresh_deployment | default(false) }}" + deployment_hash: "{{ current_deployment_hash | default('none') }}" + user_hash: "{{ current_user_secrets_hash | default('none') }}" + lock_file_stat: "{{ lock_file_stat | default({}) }}" + vdi_setup_status: "{{ vdi_setup_status | default({}) }}" + +- name: Debug role variables + ansible.builtin.debug: + msg: | + Secret Manager role variables: + - role_should_run: {{ role_should_run }} + - is_fresh_deployment: {{ is_fresh_deployment }} + - deployment_hash: {{ deployment_hash }} + - user_hash: {{ user_hash }} + - execution_reason: {{ 'Fresh deployment' if is_fresh_deployment else ('Hash mismatch' if not role_should_run else 'Normal role execution') }} + when: debug # Skip all tasks if role should not run - name: Skip secret_manager tasks if role should not run @@ -37,131 +52,132 @@ # Run secret manager tasks only if needed - name: Run secret manager tasks block: - # General secret handling - - name: Generate random database password - ansible.builtin.set_fact: - database_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}" - - - name: Get access token for Secret Manager API - ansible.builtin.uri: - url: "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" - method: GET - headers: - Metadata-Flavor: Google - return_content: true - register: access_token_result - - # Webapp admin password handling - - name: Fetch existing webapp admin password from Secret Manager - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/webapp-server-password-{{ deployment_name }}/versions/latest:access" - method: GET - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - return_content: true - register: webapp_admin_sm_result - failed_when: false - - - name: Set webapp admin password from Secret Manager (if available and not resetting) - ansible.builtin.set_fact: - webapp_admin_password: "{{ webapp_admin_sm_result.json.payload.data | b64decode }}" - webapp_password_was_fetched: true - when: - - webapp_admin_sm_result.status == 200 - - not (reset_webapp_admin_password | default(false)) - - - name: Generate new webapp admin password (if not available from Secret Manager or reset requested) - ansible.builtin.set_fact: - webapp_admin_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}" - webapp_password_was_fetched: false - when: - - webapp_admin_sm_result.status != 200 or (reset_webapp_admin_password | default(false)) - - - name: Compute Webapp admin password hash to save to Webapp server - ansible.builtin.set_fact: - webapp_admin_hash: "{{ webapp_admin_password | hash('sha256') | upper }}" - - - name: Create a GCP secret for webapp server login - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets?secretId=webapp-server-password-{{ deployment_name }}" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - replication: - automatic: {} - status_code: [200, 409] # 409 = already exists - register: webapp_secret_create - - - name: Add webapp password version to secret (only if new password was generated) - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/webapp-server-password-{{ deployment_name }}:addVersion" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - payload: - data: "{{ webapp_admin_password | b64encode }}" - register: webapp_secret - when: not (webapp_password_was_fetched | default(false)) - - - name: Debug webapp admin password handling - ansible.builtin.debug: - msg: | - Webapp admin password handling (secret_manager): - - reset_requested: {{ reset_webapp_admin_password | default(false) }} - - is_fresh_deployment: {{ is_fresh_deployment | default(false) }} - - reset_applied: {{ reset_webapp_admin_password | default(false) }} - - secret_manager_status: {{ webapp_admin_sm_result.status }} - - password_source: {{ 'Secret Manager' if webapp_admin_sm_result.status == 200 and not (reset_webapp_admin_password | default(false)) else 'Generated' }} - - password_was_fetched: {{ webapp_password_was_fetched | default(false) }} - - password_length: {{ webapp_admin_password | length if webapp_admin_password is defined else 'N/A' }} - - stored_in_secret_manager: {{ not (webapp_password_was_fetched | default(false)) }} - when: debug | default(false) - - - name: Create a GCP secret for database password - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets?secretId=db-password-{{ deployment_name }}" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - replication: - automatic: {} - status_code: [200, 409] # 409 = already exists - register: database_secret_create - - - name: Add database password version to secret - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/db-password-{{ deployment_name }}:addVersion" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - payload: - data: "{{ database_password | b64encode }}" - register: database_secret - - # Per-user secret handling - - name: Generate per-user secrets and enrich list - include_tasks: user_secret_tasks.yaml - loop: "{{ vdi_users }}" - loop_control: - loop_var: item - - - name: Set enriched vdi_users_updated variable - ansible.builtin.set_fact: - vdi_users_updated: "{{ vdi_users_updated }}" - when: vdi_users_updated is defined + # General secret handling + - name: Generate random database password + ansible.builtin.set_fact: + database_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}" + + - name: Get access token for Secret Manager API + ansible.builtin.uri: + url: "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" + method: GET + headers: + Metadata-Flavor: Google + return_content: true + register: access_token_result + + # Webapp admin password handling + - name: Fetch existing webapp admin password from Secret Manager + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/webapp-server-password-{{ deployment_name }}/versions/latest:access" + method: GET + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + return_content: true + register: webapp_admin_sm_result + failed_when: false + + - name: Set webapp admin password from Secret Manager (if available and not resetting) + ansible.builtin.set_fact: + webapp_admin_password: "{{ webapp_admin_sm_result.json.payload.data | b64decode }}" + webapp_password_was_fetched: true + when: + - webapp_admin_sm_result.status == 200 + - not (reset_webapp_admin_password | default(false)) + + - name: Generate new webapp admin password (if not available from Secret Manager or reset requested) + ansible.builtin.set_fact: + webapp_admin_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}" + webapp_password_was_fetched: false + when: + - webapp_admin_sm_result.status != 200 or (reset_webapp_admin_password | default(false)) + + - name: Compute Webapp admin password hash to save to Webapp server + ansible.builtin.set_fact: + webapp_admin_hash: "{{ webapp_admin_password | hash('sha256') | upper }}" + + - name: Create a GCP secret for webapp server login + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets?secretId=webapp-server-password-{{ deployment_name }}" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + replication: + automatic: {} + status_code: [200, 409] # 409 = already exists + register: webapp_secret_create + + - name: Add webapp password version to secret (only if new password was generated) + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/webapp-server-password-{{ deployment_name }}:addVersion" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + payload: + data: "{{ webapp_admin_password | b64encode }}" + register: webapp_secret + when: not (webapp_password_was_fetched | default(false)) + + - name: Debug webapp admin password handling + ansible.builtin.debug: + msg: | + Webapp admin password handling (secret_manager): + - reset_requested: {{ reset_webapp_admin_password | default(false) }} + - is_fresh_deployment: {{ is_fresh_deployment | default(false) }} + - reset_applied: {{ reset_webapp_admin_password | default(false) }} + - secret_manager_status: {{ webapp_admin_sm_result.status }} + - password_source: {{ 'Secret Manager' if webapp_admin_sm_result.status == 200 and not (reset_webapp_admin_password | default(false)) else 'Generated' }} + - password_was_fetched: {{ webapp_password_was_fetched | default(false) }} + - password_length: {{ webapp_admin_password | length if webapp_admin_password is defined else 'N/A' }} + - stored_in_secret_manager: {{ not (webapp_password_was_fetched | default(false)) }} + when: debug + + - name: Create a GCP secret for database password + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets?secretId=db-password-{{ deployment_name }}" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + replication: + automatic: {} + status_code: [200, 409] # 409 = already exists + register: database_secret_create + + - name: Add database password version to secret + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/db-password-{{ deployment_name }}:addVersion" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + payload: + data: "{{ database_password | b64encode }}" + register: database_secret + + # Per-user secret handling + - name: Generate per-user secrets and enrich list + include_tasks: user_secret_tasks.yaml + loop: "{{ vdi_users }}" + loop_control: + loop_var: item + + - name: Set enriched vdi_users_updated variable + ansible.builtin.set_fact: + vdi_users_updated: "{{ vdi_users_updated }}" + when: vdi_users_updated is defined + when: role_should_run # Mark role as completed diff --git a/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml b/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml index a52d694b34..499dbc7b84 100644 --- a/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml +++ b/community/modules/scripts/vdi-setup/roles/secret_manager/tasks/user_secret_tasks.yaml @@ -11,6 +11,25 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# ============================================================================= +# USER SECRET MANAGEMENT TASKS +# ============================================================================= +# +# This file handles user password management with Secret Manager integration. +# +# Password Sources: +# - "secret": Use existing Secret Manager secret (auto-creates if missing) +# - "password": Use provided password value (not currently used) +# - "generate": Generate random password +# +# Auto-Creation Behavior: +# - If a secret is referenced but doesn't exist, it will be automatically created +# - A secure random password will be generated and stored in the new secret +# - This ensures deployments never fail due to missing secrets +# - Auto-created secrets are treated as password changes for user provisioning +# +# ============================================================================= --- - name: Compute default secret_name if none provided @@ -38,7 +57,7 @@ - password_source: "{{ password_source }}" - secret_name: {{ item.secret_name | default('not set') }} - password: {{ item.password | default('not set') }} - when: debug | default(false) + when: debug - name: Fetch existing Secret Manager password for user {{ item.username }} ansible.builtin.uri: @@ -58,48 +77,85 @@ Secret Manager fetch for {{ item.username }}: - status: {{ sm_result.status }} - success: {{ sm_result.status == 200 }} - - user_config_changed: {{ user_config_changed }} + - user_hash_matches: {{ user_hash_matches | default(false) }} - password_source: {{ password_source }} - when: debug | default(false) and password_source == "secret" + when: debug and password_source == "secret" - name: Set VDI user password from Secret Manager ansible.builtin.set_fact: vdiuser_password: "{{ sm_result.json.payload.data | b64decode }}" when: password_source == "secret" and sm_result.status == 200 +- name: Handle missing secret - auto-create with generated password + ansible.builtin.debug: + msg: | + INFO: Secret Manager secret not found for {{ item.username }}. + Auto-creating secret '{{ item.secret_name }}' with generated password. + This ensures the deployment continues without errors. + when: password_source == "secret" and sm_result.status != 200 + +- name: Generate password for auto-created secret + ansible.builtin.set_fact: + vdiuser_password: "{{ lookup('password', '/dev/null length=16 chars=ascii_letters,digits') }}" + when: password_source == "secret" and sm_result.status != 200 + +- name: Create missing secret for user {{ item.username }} + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets?secretId={{ item.secret_name }}" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + replication: + automatic: {} + status_code: [200, 409] # 409 = already exists + when: password_source == "secret" and sm_result.status != 200 + register: secret_creation + +- name: Store generated password in auto-created secret + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets/{{ item.secret_name }}:addVersion" + method: POST + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + body_format: json + body: + payload: + data: "{{ vdiuser_password | b64encode }}" + when: password_source == "secret" and sm_result.status != 200 + +- name: Debug auto-created secret for {{ item.username }} + ansible.builtin.debug: + msg: | + Auto-created secret for {{ item.username }}: + - secret_name: {{ item.secret_name }} + - secret_created: {{ secret_creation.status == 200 if secret_creation is defined else 'N/A' }} + - password_generated: {{ vdiuser_password is defined }} + - password_length: {{ vdiuser_password | length if vdiuser_password is defined else 'N/A' }} + when: debug and password_source == "secret" and sm_result.status != 200 + - name: Check if password should be forced to update (only on existing deployments) ansible.builtin.set_fact: password_force_update: "{{ item.reset_password | default(false) and not (is_fresh_deployment | default(false)) }}" - name: Check if password has actually changed for {{ item.username }} ansible.builtin.set_fact: - password_actually_changed: "{{ user_config_changed or password_force_update }}" + password_actually_changed: "{{ user_hash_matches | default(true) or (item.reset_password | default(false) and not (is_fresh_deployment | default(false))) }}" when: password_source == "secret" and sm_result.status == 200 -- name: Handle failed Secret Manager fetch with reset_password (only on existing deployments) - ansible.builtin.debug: - msg: | - WARNING: Failed to fetch password from Secret Manager for {{ item.username }} - but reset_password is set. This will trigger password regeneration. - Secret: {{ item.secret_name }} - Status: {{ sm_result.status }} - when: password_source == "secret" and sm_result.status != 200 and item.reset_password | default(false) and not (is_fresh_deployment | default(false)) - -- name: Set password_actually_changed for failed Secret Manager fetch with reset_password +- name: Set password_actually_changed for auto-created secrets ansible.builtin.set_fact: password_actually_changed: true - when: password_source == "secret" and sm_result.status != 200 and item.reset_password | default(false) and not (is_fresh_deployment | default(false)) - -- name: Generate new password for user with reset_password when Secret Manager fetch fails - ansible.builtin.set_fact: - vdiuser_password: "{{ lookup('password', '/dev/null length=16 chars=ascii_letters,digits') }}" - when: password_source == "secret" and sm_result.status != 200 and item.reset_password | default(false) and not (is_fresh_deployment | default(false)) + when: password_source == "secret" and sm_result.status != 200 - name: Debug password change detection for {{ item.username }} ansible.builtin.debug: msg: | Password change detection for {{ item.username }}: - - user_config_changed: {{ user_config_changed }} + - user_hash_matches: {{ user_hash_matches | default(false) }} - password_source: {{ password_source }} - sm_result_status: {{ sm_result.status }} - reset_password: {{ item.reset_password | default(false) }} @@ -107,7 +163,7 @@ - password_force_update: {{ password_force_update }} - password_actually_changed: {{ password_actually_changed | default(false) }} - vdiuser_password_set: {{ vdiuser_password is defined }} - when: debug | default(false) and password_source == "secret" + when: debug and password_source == "secret" - name: Generate random password for user {{ item.username }} ansible.builtin.set_fact: @@ -121,7 +177,7 @@ - name: Set password_actually_changed for generate and password sources ansible.builtin.set_fact: - password_actually_changed: "{{ user_config_changed or (item.reset_password | default(false) and not (is_fresh_deployment | default(false))) }}" + password_actually_changed: "{{ not user_hash_matches | default(true) or (item.reset_password | default(false) and not (is_fresh_deployment | default(false))) }}" when: password_source in ["generate", "password"] - name: Debug final password decision for {{ item.username }} @@ -129,13 +185,13 @@ msg: | Final password decision for {{ item.username }}: - password_source: {{ password_source }} - - user_config_changed: {{ user_config_changed }} + - user_hash_matches: {{ user_hash_matches | default(false) }} - reset_password: {{ item.reset_password | default(false) }} - is_fresh_deployment: {{ is_fresh_deployment | default(false) }} - password_actually_changed: {{ password_actually_changed | default(false) }} - vdiuser_password_set: {{ vdiuser_password is defined }} - password_length: {{ vdiuser_password | length if vdiuser_password is defined else 'N/A' }} - when: debug | default(false) + when: debug - name: Create secret for user {{ item.username }} (provided password) ansible.builtin.uri: @@ -191,18 +247,8 @@ data: "{{ vdiuser_password | b64encode }}" when: password_source == "generate" -- name: Update Secret Manager for user with reset_password when Secret Manager fetch failed - ansible.builtin.uri: - url: "https://secretmanager.googleapis.com/v1/projects/{{ item.secret_project | default(project_id) }}/secrets/{{ item.secret_name }}:addVersion" - method: POST - headers: - Authorization: "Bearer {{ access_token_result.json.access_token }}" - Content-Type: "application/json" - body_format: json - body: - payload: - data: "{{ vdiuser_password | b64encode }}" - when: password_source == "secret" and sm_result.status != 200 and item.reset_password | default(false) +# Note: Auto-creation logic above handles all missing secret scenarios +# This includes both fresh deployments and reset_password scenarios - name: Set VNC configuration for user ansible.builtin.set_fact: diff --git a/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml index 12063aa9a3..e28436386d 100644 --- a/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml +++ b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/local_users.yaml @@ -19,31 +19,55 @@ key: "{{ item.username }}" register: user_exists_check ignore_errors: true - when: debug | default(false) + when: debug - name: Set user exists status for debug ansible.builtin.set_fact: user_exists_status: false - when: debug | default(false) and user_exists_check is failed + when: debug and user_exists_check is failed - name: Set user exists status for debug (success case) ansible.builtin.set_fact: user_exists_status: "{{ user_exists_check.ansible_facts.getent_passwd is defined }}" - when: debug | default(false) and user_exists_check is not failed + when: debug and user_exists_check is not failed - name: Debug user provision for {{ item.username }} ansible.builtin.debug: msg: | Provisioning user {{ item.username }}: - - user_secrets_changed: {{ user_secrets_changed }} - - password_actually_changed: {{ password_actually_changed | default(false) }} - - password_update_mode: {{ 'always' if password_actually_changed | default(user_secrets_changed) else 'on_create' }} + - is_fresh_deployment: {{ is_fresh_deployment }} + - user_hash: {{ user_hash }} + - password_update_mode: {{ 'always' if is_fresh_deployment else 'on_create' }} - user_exists: {{ user_exists_status | default(false) }} - when: debug | default(false) + when: debug - name: Provision VDI users (custom or generated passwords) block: + # Handle group changes for existing users + - name: Debug group change detection for {{ item.username }} + ansible.builtin.debug: + msg: | + Group change detection for {{ item.username }}: + - old_group: {{ vdi_setup_status.vdi_user_group | default('not_set') }} + - new_group: {{ vdi_user_group }} + - group_changed: {{ vdi_setup_status.vdi_user_group is defined and vdi_setup_status.vdi_user_group != vdi_user_group }} + when: + - debug + - not is_fresh_deployment + - vdi_setup_status.vdi_user_group is defined + + - name: Remove users from old VDI group (if group changed) + ansible.builtin.user: + name: "{{ item.username }}" + groups: "{{ vdi_setup_status.vdi_user_group }}" + remove: yes + when: + - not is_fresh_deployment + - vdi_setup_status.vdi_user_group is defined + - vdi_setup_status.vdi_user_group != vdi_user_group + ignore_errors: true + # Create the Linux user with a hashed password - name: Create VDI user {{ item.username }} ansible.builtin.user: @@ -53,7 +77,7 @@ create_home: yes append: true password: "{{ item.password | password_hash('sha512') }}" - update_password: "{{ 'always' if password_actually_changed | default(user_secrets_changed) else 'on_create' }}" + update_password: "{{ 'always' if is_fresh_deployment else 'on_create' }}" # SSH key setup - name: Ensure .ssh directory for {{ item.username }} @@ -77,25 +101,3 @@ user: "{{ item.username }}" key: "{{ ssh_key.public_key }}" state: present - -- name: Get users to remove from lock file - ansible.builtin.set_fact: - users_to_remove: "{{ vdi_setup_status.user_management.removed_users | default([]) }}" - when: lock_file_stat.stat.exists - -- name: Debug local user cleanup - ansible.builtin.debug: - msg: | - Local user cleanup: - - users_to_remove: {{ users_to_remove }} - when: debug | default(false) and lock_file_stat.stat.exists - -- name: Remove local user accounts - ansible.builtin.user: - name: "{{ item }}" - state: absent - remove: yes - loop: "{{ users_to_remove }}" - when: - - lock_file_stat.stat.exists - - users_to_remove | length > 0 diff --git a/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml index dc841d507b..33c72751ed 100644 --- a/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/user_provision/tasks/main.yaml @@ -22,11 +22,28 @@ ansible.builtin.import_role: name: lock_manager tasks_from: check_lock - register: lock_check_result -- name: Set role_should_run from lock check result +- name: Set role variables from lock manager facts ansible.builtin.set_fact: - role_should_run: "{{ lock_check_result.ansible_facts.role_should_run | default(true) }}" + role_should_run: "{{ role_should_run | default(false) }}" + is_fresh_deployment: "{{ is_fresh_deployment | default(false) }}" + deployment_hash: "{{ current_deployment_hash | default('none') }}" + user_hash: "{{ current_user_secrets_hash | default('none') }}" + lock_file_stat: "{{ lock_file_stat | default({}) }}" + vdi_setup_status: "{{ vdi_setup_status | default({}) }}" + removed_users: "{{ removed_users | default([]) }}" + vdi_user_group: "{{ vdi_user_group | default('vdiusers') }}" + +- name: Debug role variables + ansible.builtin.debug: + msg: | + User Provision role variables: + - role_should_run: {{ role_should_run }} + - is_fresh_deployment: {{ is_fresh_deployment }} + - deployment_hash: {{ deployment_hash }} + - user_hash: {{ user_hash }} + - execution_reason: {{ 'Fresh deployment' if is_fresh_deployment else ('Hash mismatch' if not role_should_run else 'Normal role execution') }} + when: debug # Skip all tasks if role should not run - name: Skip user_provision tasks if role should not run @@ -37,7 +54,6 @@ # Run user provision tasks only if needed - name: Run user provision tasks block: - - name: Ensure 'wheel' group is present group: name: wheel @@ -64,6 +80,42 @@ loop_var: item when: user_provision | lower == 'os_login' + # ============================================================================= + # USER REMOVAL HANDLING + # ============================================================================= + + - name: Handle removed users (lock local accounts) + block: + - name: Lock local user accounts for removed users + ansible.builtin.user: + name: "{{ item }}" + password: "!" + loop: "{{ removed_users }}" + when: removed_users is defined and removed_users | length > 0 + + - name: Remove users from VDI group + ansible.builtin.user: + name: "{{ item }}" + groups: "{{ vdi_user_group }}" + remove: yes + loop: "{{ removed_users }}" + when: removed_users is defined and removed_users | length > 0 + + - name: Debug user removal actions + ansible.builtin.debug: + msg: | + User removal actions completed: + {% if removed_users is defined and removed_users | length > 0 %} + - Users locked: {{ removed_users }} + - Local accounts locked with '!' password + - Users removed from {{ vdi_user_group }} group + {% else %} + - No users to remove + {% endif %} + when: debug + + when: removed_users is defined and removed_users | length > 0 + when: role_should_run # Mark role as completed diff --git a/community/modules/scripts/vdi-setup/roles/vdi_monitor/README.md b/community/modules/scripts/vdi-setup/roles/vdi_monitor/README.md index d94bf42b9e..54971254dc 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_monitor/README.md +++ b/community/modules/scripts/vdi-setup/roles/vdi_monitor/README.md @@ -11,18 +11,19 @@ The `vdi_monitor` role sets up and configures the VDI Configuration Monitor syst ## Features -### šŸ” Change Detection +### Change Detection - **File Monitoring**: Detects changes to `vars.yaml`, `install.yaml`, and `roles.tar.gz` in GCS bucket - **Password Reset Detection**: Detects `reset_password: true` flags in configuration - **Immediate Initialization**: Refreshes file tracking on startup to capture current bucket state -### āš” Automatic Reconfiguration +### Automatic Reconfiguration - **Immediate Action**: Triggers reconfiguration as soon as changes are detected - **Cooldown**: Prevents excessive reconfigurations with a 1-minute cooldown period - **Idempotent**: Safe to run multiple times without side effects - **Smart Container Management**: User changes handled via SQL updates, full container recreation only when necessary +- **Status Tracking**: Updates deployment status to `failed` when Ansible exits with non-zero code -### šŸ› ļø Modular Architecture +### Modular Architecture - **Config Module**: Constants, logging, and initialization - **File Monitor**: GCS file detection and synchronization - **Change Detector**: Change detection logic and reconfiguration @@ -30,7 +31,7 @@ The `vdi_monitor` role sets up and configures the VDI Configuration Monitor syst ## Role Structure -``` +```text vdi_monitor/ ā”œā”€ā”€ defaults/ ā”‚ ā””ā”€ā”€ main.yaml # Default variables @@ -43,8 +44,6 @@ vdi_monitor/ ā”‚ ā”œā”€ā”€ vdi-monitor-detector.sh.j2 # Change detection module ā”‚ ā”œā”€ā”€ vdi-monitor-test.sh.j2 # Testing module ā”‚ ā””ā”€ā”€ vdi-monitor.service.j2 # Systemd service -ā”œā”€ā”€ meta/ -ā”‚ ā””ā”€ā”€ main.yaml # Role metadata ā””ā”€ā”€ README.md # This file ``` @@ -74,73 +73,29 @@ The role is automatically included in the VDI setup playbook: # ... other configuration ``` -### Manual Role Execution -```bash -# Run the role manually -ansible-playbook -i localhost, --connection=local \ - -e "role=vdi_monitor" \ - /opt/vdi-setup/install.yaml -``` - ## Monitoring Commands ### Test Mode + ```bash # Run comprehensive tests sudo /usr/local/bin/vdi-monitor.sh --test ``` ### Diagnostics + ```bash # Run system diagnostics sudo /usr/local/bin/vdi-monitor.sh --diagnostics ``` ### Help + ```bash # Show help and usage sudo /usr/local/bin/vdi-monitor.sh --help ``` -## Service Management - -### Check Service Status -```bash -systemctl status vdi-monitor -``` - -### View Logs -```bash -# Monitor logs -tail -f /var/log/vdi-monitor.log - -# Ansible reconfiguration logs -tail -f /var/log/ansible-vdi-reconfig.log -``` - -### Manual Service Control -```bash -# Start service -sudo systemctl start vdi-monitor - -# Stop service -sudo systemctl stop vdi-monitor - -# Restart service -sudo systemctl restart vdi-monitor - -# Enable/disable service -sudo systemctl enable vdi-monitor -sudo systemctl disable vdi-monitor -``` - -## Dependencies - -- `lock_manager` role (for deployment locking) -- Google Cloud SDK (`gcloud`, `gsutil`) -- Ansible (`ansible-playbook`) -- Systemd (for service management) - ## Files Created ### Scripts @@ -169,32 +124,23 @@ sudo systemctl disable vdi-monitor 2. **Immediate Response**: When changes are detected, triggers reconfiguration immediately 3. **File Sync**: Downloads updated files from GCS bucket 4. **Ansible Execution**: Runs the VDI setup playbook with updated configuration -5. **Cooldown**: Prevents reconfigurations for 1 minute after successful execution - -### Password Reset Workflow -1. **Blueprint Update**: User adds `reset_password: true` to a user in blueprint -2. **Terraform Deployment**: New `vars.yaml` is uploaded to GCS with new file suffix -3. **Change Detection**: VDI monitor detects the new file suffix -4. **Reconfiguration**: Ansible roles re-run with `reset_password: true` flag -5. **Password Update**: New password is generated and stored in Secret Manager -6. **Container Preservation**: User changes are handled via SQL updates without recreating containers +5. **Status Update**: Sets deployment status to `available` on success or `failed` on error +6. **Cooldown**: Prevents reconfigurations for 1 minute after execution ## Troubleshooting -### Common Issues - -1. **Service won't start** - - Check logs: `journalctl -u vdi-monitor` - - Verify permissions: `ls -la /usr/local/bin/vdi-monitor.sh` - - Check dependencies: `which gcloud gsutil ansible-playbook` - -2. **No changes detected** - - Run test mode: `sudo /usr/local/bin/vdi-monitor.sh --test` - - Check bucket access: `gsutil ls gs://your-bucket-name/` - - Verify file tracking: Check `.current_*_file` files in `/opt/vdi-setup/` - - Check bucket name retrieval: Look for "DEBUG: Bucket name" in logs - -3. **Ansible not running after change detection** - - Check Ansible logs: `tail -f /var/log/ansible-vdi-reconfig.log` - - Verify files exist: `ls -la /opt/vdi-setup/install.yaml /opt/vdi-setup/vars.yaml` - - Run diagnostics: `sudo /usr/local/bin/vdi-monitor.sh --diagnostics` +### Service won't start +- Check logs: `journalctl -u vdi-monitor` +- Verify permissions: `ls -la /usr/local/bin/vdi-monitor.sh` +- Check dependencies: `which gcloud gsutil ansible-playbook` + +### No changes detected +- Run test mode: `sudo /usr/local/bin/vdi-monitor.sh --test` +- Check bucket access: `gsutil ls gs://your-bucket-name/` +- Verify file tracking: Check `.current_*_file` files in `/opt/vdi-setup/` +- Check bucket name retrieval: Look for "DEBUG: Bucket name" in logs + +### Ansible not running after change detection +- Check Ansible logs: `tail -f /var/log/ansible-vdi-reconfig.log` +- Verify files exist: `ls -la /opt/vdi-setup/install.yaml /opt/vdi-setup/vars.yaml` +- Run diagnostics: `sudo /usr/local/bin/vdi-monitor.sh --diagnostics` diff --git a/community/modules/scripts/vdi-setup/roles/vdi_monitor/defaults/main.yaml b/community/modules/scripts/vdi-setup/roles/vdi_monitor/defaults/main.yaml index 7cf155be2e..7bd4745f09 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_monitor/defaults/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/vdi_monitor/defaults/main.yaml @@ -17,9 +17,8 @@ # Monitoring configuration vdi_monitor_check_interval: 60 # seconds -vdi_monitor_reconfig_cooldown: 60 # seconds (1 minute) +vdi_monitor_reconfig_cooldown: 180 # seconds (3 minutes) # Logging vdi_monitor_log_file: "/var/log/vdi-monitor.log" vdi_monitor_ansible_log_file: "/var/log/ansible-vdi-reconfig.log" - \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/roles/vdi_monitor/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/vdi_monitor/tasks/main.yaml index ed932263bc..708c1f365a 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_monitor/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/vdi_monitor/tasks/main.yaml @@ -26,15 +26,35 @@ ansible.builtin.import_role: name: lock_manager tasks_from: check_lock - register: lock_check_result - name: DEBUG Finished import_role for vdi_monitor debug: msg: "Finished import_role for vdi_monitor" -- name: Set role_should_run from lock check result +- name: Set role variables from lock manager facts ansible.builtin.set_fact: - role_should_run: "{{ lock_check_result.ansible_facts.role_should_run | default(true) }}" + role_should_run: "{{ role_should_run | default(false) }}" + is_fresh_deployment: "{{ is_fresh_deployment | default(false) }}" + deployment_hash: "{{ current_deployment_hash | default('none') }}" + user_hash: "{{ current_user_secrets_hash | default('none') }}" + lock_file_stat: "{{ lock_file_stat | default({}) }}" + vdi_setup_status: "{{ vdi_setup_status | default({}) }}" + +- name: Debug role variables + ansible.builtin.debug: + msg: | + VDI Monitor role variables: + - role_should_run: {{ role_should_run }} + - is_fresh_deployment: {{ is_fresh_deployment }} + - deployment_hash: {{ deployment_hash }} + - user_hash: {{ user_hash }} + - execution_reason: {{ 'Fresh deployment' if is_fresh_deployment else ('Hash mismatch' if not role_should_run else 'Normal role execution') }} + when: debug + +- name: Set VDI monitor variables + ansible.builtin.set_fact: + vdi_monitor_reconfig_cooldown: "{{ vdi_monitor_reconfig_cooldown | default(300) }}" + vdi_monitor_check_interval: "{{ vdi_monitor_check_interval | default(60) }}" # Skip all tasks if role should not run - name: Skip vdi_monitor tasks if role should not run @@ -44,7 +64,11 @@ - name: Debug VDI monitor role execution ansible.builtin.debug: - msg: "VDI monitor role started - operation: {{ monitor_operation | default('setup') }}" + msg: | + VDI monitor role started: + - role_should_run: {{ role_should_run }} + - is_fresh_deployment: {{ is_fresh_deployment | default(false) }} + - lock_file_exists: {{ lock_file_stat.stat.exists | default(false) }} when: role_should_run - name: Ensure VDI setup directory exists @@ -66,7 +90,7 @@ - name: Debug directory creation status ansible.builtin.debug: msg: "VDI dir created: {{ vdi_dir_creation.changed }}, Monitor lib created: {{ monitor_lib_creation.changed }}" - when: debug | default(false) and role_should_run + when: debug and role_should_run # Copy VDI monitor modules - name: Copy VDI monitor configuration module @@ -74,6 +98,8 @@ src: vdi-monitor-config.sh.j2 dest: /opt/vdi-setup/vdi-monitor-lib/config.sh mode: '0755' + owner: root + group: root register: config_module_copy when: role_should_run @@ -82,6 +108,8 @@ src: vdi-monitor-file.sh.j2 dest: /opt/vdi-setup/vdi-monitor-lib/file-monitor.sh mode: '0755' + owner: root + group: root register: file_module_copy when: role_should_run @@ -90,6 +118,8 @@ src: vdi-monitor-detector.sh.j2 dest: /opt/vdi-setup/vdi-monitor-lib/change-detector.sh mode: '0755' + owner: root + group: root register: detector_module_copy when: role_should_run @@ -98,6 +128,8 @@ src: vdi-monitor-test.sh.j2 dest: /opt/vdi-setup/vdi-monitor-lib/test-utils.sh mode: '0755' + owner: root + group: root register: test_module_copy when: role_should_run @@ -106,6 +138,8 @@ src: vdi-monitor.sh.j2 dest: /usr/local/bin/vdi-monitor.sh mode: '0755' + owner: root + group: root register: main_script_copy when: role_should_run @@ -114,32 +148,52 @@ src: vdi-monitor.service.j2 dest: /etc/systemd/system/vdi-monitor.service mode: '0644' + owner: root register: service_copy when: role_should_run - name: Debug module copy status ansible.builtin.debug: msg: "VDI monitor modules copied - config: {{ config_module_copy.changed }}, file: {{ file_module_copy.changed }}, detector: {{ detector_module_copy.changed }}, test: {{ test_module_copy.changed }}, main: {{ main_script_copy.changed }}, service: {{ service_copy.changed }}" - when: debug | default(false) and role_should_run + when: debug and role_should_run # Enable and start VDI monitor service - name: Reload systemd daemon - ansible.builtin.systemd: + systemd: daemon_reload: yes - when: service_copy.changed and role_should_run + when: role_should_run and service_copy.changed + register: daemon_reload_result + +- name: Check if VDI monitor service is already running + systemd: + name: vdi-monitor + register: monitor_service_status + when: role_should_run + +- name: Debug VDI monitor service status check + ansible.builtin.debug: + msg: "VDI monitor service status - Active: {{ monitor_service_status.status.ActiveState }}, Enabled: {{ monitor_service_status.status.LoadState }}" + when: debug and role_should_run - name: Enable VDI monitor service - ansible.builtin.systemd: + systemd: name: vdi-monitor enabled: yes state: started register: monitor_service_start - when: role_should_run + when: role_should_run and not monitor_service_status.status.ActiveState == "active" + +- name: Set service start result for already running service + ansible.builtin.set_fact: + monitor_service_start: + changed: false + status: "{{ monitor_service_status.status }}" + when: role_should_run and monitor_service_status.status.ActiveState == "active" - name: Debug VDI monitor service status ansible.builtin.debug: msg: "VDI monitor service started: {{ monitor_service_start.changed }}" - when: debug | default(false) and role_should_run + when: debug and role_should_run - name: Debug VDI monitor role completion ansible.builtin.debug: @@ -158,4 +212,3 @@ name: lock_manager tasks_from: create_lock when: role_should_run - \ No newline at end of file diff --git a/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-config.sh.j2 b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-config.sh.j2 index 9df33ef5b7..912eb6d468 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-config.sh.j2 +++ b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-config.sh.j2 @@ -87,7 +87,7 @@ validate_environment() { log_message "ERROR: gcloud command not found" return 1 fi - + # Check if gsutil is available if ! command -v gsutil >/dev/null 2>&1; then log_message "ERROR: gsutil command not found" @@ -135,4 +135,4 @@ initialize_monitoring() { log_message "Using bucket: gs://$BUCKET_NAME" log_message "VDI monitoring system initialized successfully" -} \ No newline at end of file +} diff --git a/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-detector.sh.j2 b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-detector.sh.j2 index 9fbcf364b3..7e6515a4fd 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-detector.sh.j2 +++ b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-detector.sh.j2 @@ -161,7 +161,7 @@ trigger_reconfig() { fi log_message "DEBUG: Running Ansible playbook from $(pwd)" - log_message "DEBUG: Command: ansible-playbook install.yaml --connection=local --inventory=localhost, --limit=localhost --extra-vars=@vars.yaml --extra-vars=debug=true" + log_message "DEBUG: Command: ansible-playbook install.yaml --connection=local --inventory=localhost, --limit=localhost --extra-vars=@vars.yaml" # Run Ansible playbook with re-configuration ansible-playbook install.yaml \ @@ -188,7 +188,7 @@ trigger_reconfig() { log_message "DEBUG: $line" done fi - update_deployment_status "available" + update_deployment_status "failed" fi return $exit_code @@ -338,4 +338,4 @@ run_monitoring_loop() { # Sleep for configured interval before next check sleep {{ vdi_monitor_check_interval | default(60) }} done -} \ No newline at end of file +} diff --git a/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-file.sh.j2 b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-file.sh.j2 index 88a66cf89b..e0587d01a7 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-file.sh.j2 +++ b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-file.sh.j2 @@ -180,4 +180,4 @@ initialize_file_tracking() { fi log_message "File tracking initialization complete" -} \ No newline at end of file +} diff --git a/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-test.sh.j2 b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-test.sh.j2 index c454c554d1..9eb94e5a38 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-test.sh.j2 +++ b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor-test.sh.j2 @@ -138,4 +138,4 @@ run_diagnostics() { log_message " gcloud available: $(command -v gcloud >/dev/null 2>&1 && echo 'yes' || echo 'no')" log_message " gsutil available: $(command -v gsutil >/dev/null 2>&1 && echo 'yes' || echo 'no')" log_message " ansible-playbook available: $(command -v ansible-playbook >/dev/null 2>&1 && echo 'yes' || echo 'no')" -} \ No newline at end of file +} diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml index 27bca2d727..2b07b55747 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/guacamole.yaml @@ -13,30 +13,6 @@ # limitations under the License. --- -# ============================================================================= -# EARLY PASSWORD RESET DETECTION -# ============================================================================= - -- name: Get users that need password reset (only on existing deployments) - ansible.builtin.set_fact: - users_needing_reset: "{{ vdi_users_updated | selectattr('reset_password', 'defined') | selectattr('reset_password', 'equalto', true) | list }}" - when: not (is_fresh_deployment | default(false)) - -- name: Set empty users_needing_reset for fresh deployments - ansible.builtin.set_fact: - users_needing_reset: [] - when: is_fresh_deployment | default(false) - -- name: Debug password reset detection - ansible.builtin.debug: - msg: | - Password reset detection: - - is_fresh_deployment: {{ is_fresh_deployment | default(false) }} - - users_needing_reset: {{ users_needing_reset | length }} - - users: {{ users_needing_reset | map(attribute='username') | list }} - - user_details: {{ users_needing_reset | map('combine', {'password_length': item.password | length if item.password is defined else 'N/A'}) | list }} - when: debug | default(false) - # ============================================================================= # DOCKER SETUP (IDEMPOTENT) # ============================================================================= @@ -111,7 +87,7 @@ - guac_db running: {{ 'guac_db' in running_containers.stdout }} - guacd running: {{ 'guacd' in running_containers.stdout }} - guac_app running: {{ 'guac_app' in running_containers.stdout }} - when: debug | default(false) + when: debug # ============================================================================= # DATABASE INITIALIZATION LOGIC @@ -128,248 +104,712 @@ - /opt/guacamole-db/data - /opt/guacamole-db/initdb -- name: Check if database needs re-initialization due to user changes - ansible.builtin.command: > - docker exec guac_db psql -U guacamole_db -d guacamole_db - -c "SELECT COUNT(*) FROM guacamole_entity WHERE type='USER' AND name != 'guacadmin';" - register: existing_users_check - changed_when: false - when: "'guac_db' in running_containers.stdout" - -- name: Debug user configuration status (basic info) +- name: Debug user configuration status ansible.builtin.debug: msg: | - Database user check: N/A (no existing database) - Current users in config: {{ vdi_users_updated | length }} - Lock file exists: {{ lock_file_stat.stat.exists }} - User secrets hash changed: {{ vdi_setup_status.user_secrets_status.user_secrets_hash != current_user_secrets_hash if lock_file_stat.stat.exists else 'N/A' }} - User secrets hash (stored): {{ vdi_setup_status.user_secrets_status.user_secrets_hash if lock_file_stat.stat.exists else 'N/A' }} - User secrets hash (current): {{ current_user_secrets_hash }} - when: debug | default(false) and not (existing_users_check is defined) - -- name: Check if user configuration has changed + User configuration status: + - is_fresh_deployment: {{ is_fresh_deployment }} + - deployment_hash: {{ deployment_hash }} + - user_hash: {{ user_hash }} + - User hash changed: {{ not is_fresh_deployment and vdi_setup_status.user_hash is defined and vdi_setup_status.user_hash != user_hash }} + when: debug + +- name: Check if users have changed (hash-based) ansible.builtin.set_fact: - users_changed: "{{ lock_file_stat.stat.exists and vdi_setup_status.user_secrets_status.user_secrets_hash is defined and vdi_setup_status.user_secrets_status.user_secrets_hash != 'none' and vdi_setup_status.user_secrets_status.user_secrets_hash != current_user_secrets_hash }}" + users_changed: "{{ not is_fresh_deployment and ((new_users is defined and new_users | length > 0) or (removed_users is defined and removed_users | length > 0) or (users_needing_reset is defined and users_needing_reset | length > 0) or reset_webapp_admin_password or (users_with_port_changes is defined and users_with_port_changes | length > 0)) }}" -- name: Check if database needs initialization +- name: Check if database needs initialization (smart detection) ansible.builtin.set_fact: - db_needs_init: "{{ force_rerun | default(false) or not lock_file_stat.stat.exists or (lock_file_stat.stat.exists and vdi_setup_status.user_secrets_status.user_secrets_hash is not defined) or (lock_file_stat.stat.exists and vdi_setup_status.user_secrets_status.user_secrets_hash == 'none') }}" + # Only reinitialize database for infrastructure changes, not user changes + db_needs_init: "{{ is_fresh_deployment or force_rerun | default(false) or infrastructure_changed }}" + +- name: Debug user change detection (from lock_manager) + ansible.builtin.debug: + msg: | + User change detection (from lock_manager): + - users_changed: {{ users_changed }} + - new_users: {{ new_users | length if new_users is defined else 0 }} + - removed_users: {{ removed_users | length if removed_users is defined else 0 }} + - new_user_names: {{ new_users | map(attribute='username') | list if new_users is defined and new_users | length > 0 else [] }} + - removed_user_names: {{ removed_users if removed_users is defined and removed_users | length > 0 else [] }} + when: debug and users_changed - name: Debug user change detection ansible.builtin.debug: msg: | User change detection: + - is_fresh_deployment: {{ is_fresh_deployment }} - users_changed: {{ users_changed }} + - infrastructure_changed: {{ infrastructure_changed }} - db_needs_init: {{ db_needs_init }} - - will_reinitialize: {{ users_changed or db_needs_init }} + - will_reinitialize: {{ db_needs_init }} + - will_apply_targeted_updates: {{ users_changed and not db_needs_init }} - users_needing_reset: {{ users_needing_reset | length }} - when: debug | default(false) + - reset_webapp_admin_password: {{ reset_webapp_admin_password }} + - users_with_port_changes: {{ users_with_port_changes | length if users_with_port_changes is defined else 0 }} + - webapp_port_changed: {{ webapp_port_changed }} + - vdi_resolution_changed: {{ vdi_resolution_changed }} + - vdi_resolution_locked_changed: {{ vdi_resolution_locked_changed }} + - new_users: {{ new_users | length if new_users is defined else 0 }} + - removed_users: {{ removed_users | length if removed_users is defined else 0 }} + - new_user_names: {{ new_users | map(attribute='username') | list if new_users is defined and new_users | length > 0 else [] }} + - removed_user_names: {{ removed_users if removed_users is defined and removed_users | length > 0 else [] }} + when: debug # ============================================================================= -# DATABASE RE-INITIALIZATION (ONLY WHEN NECESSARY) +# TARGETED USER MANAGEMENT (NO CONTAINER RECREATION) # ============================================================================= -- name: Re-initialize database if database needs initialization +- name: Handle user changes without container recreation block: - - name: Enable pgcrypto extension for initdb - ansible.builtin.copy: - dest: /opt/guacamole-db/initdb/00-enable-pgcrypto.sql - owner: root - group: root - mode: '0644' - content: | - -- Make digest(), hmac(), etc. available - CREATE EXTENSION IF NOT EXISTS pgcrypto; - - - name: Template per-user Guacamole user SQL - ansible.builtin.template: - src: guacamole/user_bootstrap.sql.j2 - dest: "/opt/guacamole-db/initdb/02-{{ item.username }}_user.sql" - owner: root - group: root - mode: '0644' - loop: "{{ vdi_users_updated }}" - loop_control: - loop_var: item - vars: - username: "{{ item.username }}" - vdi_password: "{{ item.password }}" - - - name: Template per-user SQL for Guacamole connections - ansible.builtin.template: - src: guacamole/connection_bootstrap.sql.j2 - dest: "/opt/guacamole-db/initdb/03-{{ item.username }}_connection.sql" - owner: root - group: root - mode: '0644' - loop: "{{ vdi_users_updated }}" - loop_control: - loop_var: item - vars: - port: "{{ item.port }}" - username: "{{ item.username }}" - vnc_password: "{{ item.vncserver_password }}" - - - name: Generate combined initdb.sql - ansible.builtin.shell: | - docker run --rm \ - -v /opt/guacamole-db/initdb:/opt/guacamole-db/initdb \ - guacamole/guacamole:latest \ - /opt/guacamole/bin/initdb.sh --postgresql \ - > /opt/guacamole-db/initdb/01-initdb.sql - args: - creates: /opt/guacamole-db/initdb/01-initdb.sql - - - name: Update guacadmin password hash in 01-initdb.sql - ansible.builtin.replace: - path: /opt/guacamole-db/initdb/01-initdb.sql - regexp: "decode\\('CA458A7D494E3BE824F5E1E175A1556C0F8EEF2C2D7DF3633BEC4A29C4411960', 'hex'\\)" - replace: "decode('{{ webapp_admin_hash }}', 'hex')" - - - name: Remove salt from guacadmin in 01-initdb.sql - ansible.builtin.replace: - path: /opt/guacamole-db/initdb/01-initdb.sql - regexp: "decode\\('FE24ADC5E11E2B25288D1704ABE67A79E342ECC26064CE69C5B3177795A82264', 'hex'\\)" - replace: "NULL" - - - name: Debug webapp admin password usage - ansible.builtin.debug: - msg: | - Webapp admin password usage (vdi_tool): - - password_provided_by_secret_manager: {{ webapp_admin_password is defined }} - - password_length: {{ webapp_admin_password | length if webapp_admin_password is defined else 'N/A' }} - - hash_computed: {{ webapp_admin_hash is defined }} - when: debug | default(false) - - - name: Stop and remove existing containers - ansible.builtin.command: docker stop {{ item }} - loop: - - guac_app - - guacd - - guac_db - ignore_errors: true - - - name: Remove existing containers - ansible.builtin.command: docker rm {{ item }} - loop: - - guac_app - - guacd - - guac_db - ignore_errors: true - - - name: Backup existing database data - ansible.builtin.shell: > - tar -czf /opt/guacamole-db/backup-$(date +%Y%m%d-%H%M%S).tar.gz - -C /opt/guacamole-db data/ - ignore_errors: true - - - name: Remove existing database data - ansible.builtin.command: rm -rf /opt/guacamole-db/data/* - ignore_errors: true - - - name: Start Guacamole PostgreSQL container (re-initialized) - ansible.builtin.command: > - docker run -d --name guac_db --network guac_net - -e POSTGRES_USER=guacamole_db - -e POSTGRES_PASSWORD={{ database_password }} - -e POSTGRES_DB=guacamole_db - -v /opt/guacamole-db/data:/var/lib/postgresql/data - -v /opt/guacamole-db/initdb:/docker-entrypoint-initdb.d - --restart always - postgres:latest - register: postgres_restart - failed_when: - - postgres_restart.rc != 0 - - "'already in use' not in postgres_restart.stderr" - - - name: Wait for PostgreSQL to be ready after re-initialization - ansible.builtin.command: docker exec guac_db pg_isready -U guacamole_db - register: postgres_ready_after_restart - until: postgres_ready_after_restart.rc == 0 - retries: 30 - delay: 2 - - - name: Check if Guacamole tables exist - ansible.builtin.command: > - docker exec guac_db psql -U guacamole_db -d guacamole_db -c - "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_name LIKE 'guacamole_%';" - register: table_check - changed_when: false - - - name: Debug table check - ansible.builtin.debug: - msg: | - Guacamole tables found: - {{ table_check.stdout }} - when: debug | default(false) - - - name: Check if connections were created - ansible.builtin.command: > - docker exec guac_db psql -U guacamole_db -d guacamole_db -c - "SELECT connection_name, protocol FROM guacamole_connection WHERE connection_name NOT LIKE '%SSH';" - register: connection_check - changed_when: false - - - name: Debug connection check - ansible.builtin.debug: - msg: | - VNC Connections found: - {{ connection_check.stdout }} - when: debug | default(false) + - name: Get enriched new users from vdi_users_updated + ansible.builtin.set_fact: + new_users_enriched: "{{ vdi_users_updated | selectattr('username', 'in', new_users | map(attribute='username') | list) | list }}" + when: users_changed and new_users is defined and new_users | length > 0 + - name: Debug user update mode + ansible.builtin.debug: + msg: | + User changes detected - applying targeted updates (no container recreation): + - users_changed: {{ users_changed }} + - db_needs_init: {{ db_needs_init }} + - force_rerun: {{ force_rerun }} + - users_needing_reset: {{ users_needing_reset | length }} + - new_users: {{ new_users | length if new_users is defined else 0 }} + - removed_users: {{ removed_users | length if removed_users is defined else 0 }} + - new_user_names: {{ new_users | map(attribute='username') | list if new_users is defined and new_users | length > 0 else [] }} + - removed_user_names: {{ removed_users if removed_users is defined and removed_users | length > 0 else [] }} + when: debug + + # ============================================================================= + # ADD NEW USERS + # ============================================================================= + + - name: Add new users to database (without container recreation) + ansible.builtin.template: + src: guacamole/user_bootstrap.sql.j2 + dest: "/opt/guacamole-db/initdb/02-{{ item.username }}_user.sql" + owner: root + group: root + mode: '0644' + loop: "{{ new_users_enriched | default(new_users) }}" + when: new_users is defined and new_users | length > 0 + vars: + username: "{{ item.username }}" + vdi_password: "{{ item.password }}" + + - name: Execute new user SQL files + ansible.builtin.command: docker exec guac_db psql -U guacamole_db -d guacamole_db -f /docker-entrypoint-initdb.d/02-{{ item.username }}_user.sql + loop: "{{ new_users_enriched | default(new_users) }}" + when: new_users is defined and new_users | length > 0 + + - name: Add new user connections to database + ansible.builtin.template: + src: guacamole/connection_bootstrap.sql.j2 + dest: "/opt/guacamole-db/initdb/03-{{ item.username }}_connection.sql" + owner: root + group: root + mode: '0644' + loop: "{{ new_users_enriched | default(new_users) }}" + when: new_users is defined and new_users | length > 0 + vars: + username: "{{ item.username }}" + port: "{{ item.port }}" + vnc_password: "{{ item.vncserver_password }}" + + - name: Execute new connection SQL files + ansible.builtin.command: docker exec guac_db psql -U guacamole_db -d guacamole_db -f /docker-entrypoint-initdb.d/03-{{ item.username }}_connection.sql + loop: "{{ new_users_enriched | default(new_users) }}" + when: new_users is defined and new_users | length > 0 + + # ============================================================================= + # REMOVE/DISABLE USERS + # ============================================================================= + + - name: Disable removed users in Guacamole database + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c + "UPDATE guacamole_user SET disabled = true WHERE entity_id IN (SELECT entity_id FROM guacamole_entity WHERE name = '{{ item }}' AND type = 'USER');" + loop: "{{ removed_users }}" + when: removed_users is defined and removed_users | length > 0 + + - name: Revoke connection permissions for removed users + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c + "DELETE FROM guacamole_connection_permission WHERE entity_id IN (SELECT entity_id FROM guacamole_entity WHERE name = '{{ item }}' AND type = 'USER');" + loop: "{{ removed_users }}" + when: removed_users is defined and removed_users | length > 0 + + - name: Delete connections for removed users + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c + "DELETE FROM guacamole_connection WHERE connection_name IN ('{{ item }}', '{{ item }} SSH');" + loop: "{{ removed_users }}" + when: removed_users is defined and removed_users | length > 0 + + # ============================================================================= + # PASSWORD RESETS + # ============================================================================= + + - name: Update user passwords in database (for reset_password users) + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c "UPDATE guacamole_user SET password_salt = NULL, password_hash = digest('{{ item.password }}', 'SHA-256'), password_date = now() FROM guacamole_entity WHERE guacamole_user.entity_id = guacamole_entity.entity_id AND guacamole_entity.name = '{{ item.username }}' AND guacamole_entity.type = 'USER';" + loop: "{{ vdi_users_updated | selectattr('reset_password', 'defined') | selectattr('reset_password', 'equalto', true) | list }}" + when: users_needing_reset | length > 0 + + # ============================================================================= + # VNC CONNECTION PASSWORD UPDATES + # ============================================================================= + + - name: Update VNC connection passwords in database (for all users) + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c "UPDATE guacamole_connection_parameter SET parameter_value = '{{ item.vncserver_password }}' FROM guacamole_connection WHERE guacamole_connection_parameter.connection_id = guacamole_connection.connection_id AND guacamole_connection.connection_name = '{{ item.username }}' AND guacamole_connection_parameter.parameter_name = 'password';" + loop: "{{ vdi_users_updated }}" + when: users_changed + + # ============================================================================= + # WEBAPP ADMIN PASSWORD RESET + # ============================================================================= + + - name: Update webapp admin password in database (if reset requested) + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c + "UPDATE guacamole_user SET password_salt = NULL, password_hash = decode('{{ webapp_admin_hash }}', 'hex'), password_date = now() FROM guacamole_entity WHERE guacamole_user.entity_id = guacamole_entity.entity_id AND guacamole_entity.name = 'guacadmin' AND guacamole_entity.type = 'USER';" + when: reset_webapp_admin_password and not is_fresh_deployment + + - name: Restart Guacamole webapp container (if webapp admin password reset) + ansible.builtin.command: docker restart guac_app + when: reset_webapp_admin_password and not is_fresh_deployment + + - name: Wait for Guacamole webapp to be ready after restart + ansible.builtin.wait_for: + host: 127.0.0.1 + port: "{{ vdi_webapp_port }}" + delay: 5 + timeout: 60 + when: reset_webapp_admin_password and not is_fresh_deployment + + # ============================================================================= + # PORT CHANGES + # ============================================================================= + + - name: Update user connection ports in database (for port changes) + ansible.builtin.template: + src: guacamole/connection_port_update.sql.j2 + dest: "/opt/guacamole-db/initdb/04-{{ item.username }}_port_update.sql" + owner: root + group: root + mode: '0644' + loop: "{{ users_with_port_changes }}" + when: users_with_port_changes is defined and users_with_port_changes | length > 0 + vars: + username: "{{ item.username }}" + port: "{{ item.port }}" + + - name: Execute port update SQL files + ansible.builtin.command: docker exec guac_db psql -U guacamole_db -d guacamole_db -f /docker-entrypoint-initdb.d/04-{{ item.username }}_port_update.sql + loop: "{{ users_with_port_changes }}" + when: users_with_port_changes is defined and users_with_port_changes | length > 0 + + - name: Debug targeted updates completion + ansible.builtin.debug: + msg: | + Targeted user updates completed: + - New users added: {{ new_users | map(attribute='username') | list if new_users is defined and new_users | length > 0 else [] }} + - Users removed: {{ removed_users if removed_users is defined and removed_users | length > 0 else [] }} + - Password resets: {{ users_needing_reset | map(attribute='username') | list if users_needing_reset is defined and users_needing_reset | length > 0 else [] }} + - Webapp admin password reset: {{ reset_webapp_admin_password and not is_fresh_deployment }} + - User port changes: {{ users_with_port_changes | map(attribute='username') | list if users_with_port_changes is defined and users_with_port_changes | length > 0 else [] }} + when: debug + + when: users_changed and not db_needs_init - when: db_needs_init +# ============================================================================= +# RESOLUTION LOCKED CHANGES +# ============================================================================= + +- name: Update connection disable-display-resize parameter in database (for resolution locked changes) + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c + "UPDATE guacamole_connection_parameter SET parameter_value = '{{ 'true' if vdi_resolution_locked else 'false' }}' WHERE connection_id IN (SELECT connection_id FROM guacamole_connection WHERE protocol = 'vnc') AND parameter_name = 'disable-display-resize';" + when: vdi_resolution_locked_changed + +- name: Debug resolution locked change completion + ansible.builtin.debug: + msg: | + Resolution locked change completed: + - vdi_resolution_locked_changed: {{ vdi_resolution_locked_changed }} + - new_resolution_locked: {{ vdi_resolution_locked }} + - previous_resolution_locked: {{ vdi_setup_status.vdi_resolution_locked | default('unknown') }} + when: debug and vdi_resolution_locked_changed # ============================================================================= -# TARGETED PASSWORD UPDATES (NO CONTAINER RECREATION) +# DATABASE PASSWORD RETRIEVAL (FOR CONTAINER RECREATION) # ============================================================================= -- name: Handle user changes without container recreation +- name: Get access token for Secret Manager API (for database password) + ansible.builtin.uri: + url: "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" + method: GET + headers: + Metadata-Flavor: Google + return_content: true + register: access_token_result + when: webapp_port_changed or db_needs_init + +- name: Fetch database password from Secret Manager + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/db-password-{{ deployment_name }}/versions/latest:access" + method: GET + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + return_content: true + register: database_password_sm_result + when: webapp_port_changed or db_needs_init + +- name: Set database password from Secret Manager + ansible.builtin.set_fact: + database_password: "{{ database_password_sm_result.json.payload.data | b64decode }}" + when: (webapp_port_changed or db_needs_init) and database_password_sm_result.status == 200 + +- name: Fetch webapp admin password from Secret Manager + ansible.builtin.uri: + url: "https://secretmanager.googleapis.com/v1/projects/{{ secret_project }}/secrets/webapp-server-password-{{ deployment_name }}/versions/latest:access" + method: GET + headers: + Authorization: "Bearer {{ access_token_result.json.access_token }}" + Content-Type: "application/json" + return_content: true + register: webapp_admin_sm_result + when: is_fresh_deployment or infrastructure_changed or webapp_port_changed + +- name: Set webapp admin password from Secret Manager + ansible.builtin.set_fact: + webapp_admin_password: "{{ webapp_admin_sm_result.json.payload.data | b64decode }}" + when: (is_fresh_deployment or infrastructure_changed or webapp_port_changed) and webapp_admin_sm_result.status == 200 + +- name: Calculate webapp admin hash from password + ansible.builtin.set_fact: + webapp_admin_hash: "{{ webapp_admin_password | hash('sha256') | upper }}" + when: webapp_admin_password is defined + +- name: Debug password retrieval + ansible.builtin.debug: + msg: | + Password retrieval: + - is_fresh_deployment: {{ is_fresh_deployment }} + - infrastructure_changed: {{ infrastructure_changed }} + - webapp_port_changed: {{ webapp_port_changed }} + - database_secret_status: {{ database_password_sm_result.status if database_password_sm_result is defined else 'N/A' }} + - webapp_secret_status: {{ webapp_admin_sm_result.status if webapp_admin_sm_result is defined else 'N/A' }} + - database_password_retrieved: {{ database_password is defined }} + - webapp_admin_password_retrieved: {{ webapp_admin_password is defined }} + - database_password_length: {{ database_password | length if database_password is defined else 'N/A' }} + - webapp_admin_password_length: {{ webapp_admin_password | length if webapp_admin_password is defined else 'N/A' }} + when: debug and (is_fresh_deployment or infrastructure_changed or webapp_port_changed) + +# ============================================================================= +# WEBAPP PORT CHANGE HANDLING +# ============================================================================= + +- name: Handle webapp port changes block: - - name: Debug user update mode - ansible.builtin.debug: - msg: | - User changes detected - applying targeted updates (no container recreation): - - users_changed: {{ users_changed }} - - db_needs_init: {{ db_needs_init }} - - force_rerun: {{ force_rerun | default(false) }} - - users_needing_reset: {{ users_needing_reset | length }} - when: debug | default(false) - - - name: Update user passwords in database (for reset_password users) - ansible.builtin.command: > - docker exec guac_db psql -U guacamole_db -d guacamole_db -c "UPDATE guacamole_user SET password_salt = NULL, password_hash = digest('{{ item.password }}', 'SHA-256'), password_date = now() FROM guacamole_entity WHERE guacamole_user.entity_id = guacamole_entity.entity_id AND guacamole_entity.name = '{{ item.username }}' AND guacamole_entity.type = 'USER';" - loop: "{{ users_needing_reset }}" - when: users_needing_reset | length > 0 - - - name: Update connection ports for users - ansible.builtin.command: > - docker exec guac_db psql -U guacamole_db -d guacamole_db -c "UPDATE guacamole_connection_parameter SET parameter_value = '{{ item.port }}' FROM guacamole_connection WHERE guacamole_connection_parameter.connection_id = guacamole_connection.connection_id AND guacamole_connection.connection_name = '{{ item.username }}' AND guacamole_connection_parameter.parameter_name = 'port';" - loop: "{{ vdi_users_updated }}" - - - name: Debug password updates - ansible.builtin.debug: - msg: | - Password updates applied: - {% for user in users_needing_reset %} - - {{ user.username }}: password updated - {% endfor %} - Total users needing reset: {{ users_needing_reset | length }} - when: debug | default(false) and users_needing_reset | length > 0 - - when: users_needing_reset | length > 0 and not db_needs_init + - name: Debug webapp port change + ansible.builtin.debug: + msg: | + Webapp port change detected: + - current_port: {{ vdi_webapp_port }} + - previous_port: {{ vdi_setup_status.webapp_port | default('none') }} + - will_recreate_container: True + - database_password_available: {{ database_password is defined }} + when: debug + + - name: Stop Guacamole webapp container (for port change) + ansible.builtin.command: docker stop guac_app + ignore_errors: true + + - name: Remove Guacamole webapp container (for port change) + ansible.builtin.command: docker rm guac_app + ignore_errors: true + + - name: Start Guacamole webapp container with new port + ansible.builtin.command: > + docker run -d --name guac_app --network guac_net + -p {{ vdi_webapp_port }}:8080 + -e POSTGRES_HOSTNAME=guac_db + -e POSTGRES_PORT=5432 + -e POSTGRES_DATABASE=guacamole_db + -e POSTGRES_USER=guacamole_db + -e POSTGRES_PASSWORD={{ database_password }} + -e POSTGRES_AUTO_CREATE_ACCOUNTS=true + -e GUACD_HOSTNAME=guacd + -e GUACD_PORT=4822 + --restart always + guacamole/guacamole:latest + + - name: Wait for Guacamole webapp to be ready after port change + ansible.builtin.wait_for: + host: 127.0.0.1 + port: "{{ vdi_webapp_port }}" + delay: 5 + timeout: 60 + + - name: Update database user password (for port change) + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c + "ALTER USER guacamole_db WITH PASSWORD '{{ database_password }}';" + + - name: Update webapp admin password in database (for port change) + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c + "UPDATE guacamole_user SET password_salt = NULL, password_hash = decode('{{ webapp_admin_password | hash('sha256') | upper }}', 'hex'), password_date = now() FROM guacamole_entity WHERE guacamole_user.entity_id = guacamole_entity.entity_id AND guacamole_entity.name = 'guacadmin' AND guacamole_entity.type = 'USER';" + + - name: Restart Guacamole webapp container (after password update) + ansible.builtin.command: docker restart guac_app + + - name: Wait for Guacamole webapp to be ready after restart + ansible.builtin.wait_for: + host: 127.0.0.1 + port: "{{ vdi_webapp_port }}" + delay: 5 + timeout: 60 + + - name: Debug webapp port change completion + ansible.builtin.debug: + msg: | + Webapp port change completed: + - new_port: {{ vdi_webapp_port }} + - container_recreated: True + - webapp_admin_password_updated: True + when: debug + + when: webapp_port_changed and not is_fresh_deployment # ============================================================================= -# CONTAINER MANAGEMENT (IDEMPOTENT) +# DATABASE RE-INITIALIZATION (ONLY WHEN NECESSARY) # ============================================================================= -- name: Wait for PostgreSQL container to be ready - ansible.builtin.wait_for: - host: 127.0.0.1 - port: 5432 +- name: Re-initialize database if database needs initialization + block: + - name: Enable pgcrypto extension for initdb + ansible.builtin.copy: + dest: /opt/guacamole-db/initdb/00-enable-pgcrypto.sql + owner: root + group: root + mode: '0644' + content: | + -- Make digest(), hmac(), etc. available + CREATE EXTENSION IF NOT EXISTS pgcrypto; + + - name: Template per-user Guacamole user SQL + ansible.builtin.template: + src: guacamole/user_bootstrap.sql.j2 + dest: "/opt/guacamole-db/initdb/02-{{ item.username }}_user.sql" + owner: root + group: root + mode: '0644' + loop: "{{ vdi_users_updated }}" + loop_control: + loop_var: item + vars: + username: "{{ item.username }}" + vdi_password: "{{ item.password }}" + + - name: Template per-user SQL for Guacamole connections + ansible.builtin.template: + src: guacamole/connection_bootstrap.sql.j2 + dest: "/opt/guacamole-db/initdb/03-{{ item.username }}_connection.sql" + owner: root + group: root + mode: '0644' + loop: "{{ vdi_users_updated }}" + loop_control: + loop_var: item + vars: + port: "{{ item.port }}" + username: "{{ item.username }}" + vnc_password: "{{ item.vncserver_password }}" + + - name: Generate combined initdb.sql + ansible.builtin.shell: | + docker run --rm \ + -v /opt/guacamole-db/initdb:/opt/guacamole-db/initdb \ + guacamole/guacamole:latest \ + /opt/guacamole/bin/initdb.sh --postgresql \ + > /opt/guacamole-db/initdb/01-initdb.sql + + - name: Debug hash replacement conditions + ansible.builtin.debug: + msg: | + Hash replacement conditions: + - is_fresh_deployment: {{ is_fresh_deployment }} + - vdi_setup_status.webapp_admin_hash is defined: {{ vdi_setup_status.webapp_admin_hash is defined }} + - vdi_setup_status.webapp_admin_hash: '{{ vdi_setup_status.webapp_admin_hash | default("NOT_DEFINED") }}' + - vdi_setup_status.webapp_admin_hash != '': {{ (vdi_setup_status.webapp_admin_hash | default('')) != '' }} + - vdi_setup_status.webapp_admin_hash != webapp_admin_hash: {{ (vdi_setup_status.webapp_admin_hash | default('')) != webapp_admin_hash }} + - webapp_admin_hash: '{{ webapp_admin_hash }}' + - (vdi_setup_status.webapp_admin_hash is not defined) or (vdi_setup_status.webapp_admin_hash == ''): {{ (vdi_setup_status.webapp_admin_hash is not defined) or ((vdi_setup_status.webapp_admin_hash | default('')) == '') }} + when: debug + + - name: Update guacadmin password hash in 01-initdb.sql (initial deployment) + ansible.builtin.replace: + path: /opt/guacamole-db/initdb/01-initdb.sql + regexp: "decode\\('CA458A7D494E3BE824F5E1E175A1556C0F8EEF2C2D7DF3633BEC4A29C4411960', 'hex'\\)" + replace: "decode('{{ webapp_admin_hash }}', 'hex')" + when: is_fresh_deployment + + - name: Update guacadmin password hash in 01-initdb.sql (force rerun) + ansible.builtin.replace: + path: /opt/guacamole-db/initdb/01-initdb.sql + regexp: "decode\\('{{ vdi_setup_status.webapp_admin_hash | default('CA458A7D494E3BE824F5E1E175A1556C0F8EEF2C2D7DF3633BEC4A29C4411960') }}', 'hex'\\)" + replace: "decode('{{ webapp_admin_hash }}', 'hex')" + when: + - not is_fresh_deployment + - vdi_setup_status.webapp_admin_hash is defined + - vdi_setup_status.webapp_admin_hash != '' + + - name: Update guacadmin password hash in 01-initdb.sql (force rerun fallback) + ansible.builtin.replace: + path: /opt/guacamole-db/initdb/01-initdb.sql + regexp: "decode\\('CA458A7D494E3BE824F5E1E175A1556C0F8EEF2C2D7DF3633BEC4A29C4411960', 'hex'\\)" + replace: "decode('{{ webapp_admin_hash }}', 'hex')" + when: + - not is_fresh_deployment + - (vdi_setup_status.webapp_admin_hash is not defined) or (vdi_setup_status.webapp_admin_hash == '') + + - name: Remove salt from guacadmin in 01-initdb.sql + ansible.builtin.replace: + path: /opt/guacamole-db/initdb/01-initdb.sql + regexp: "decode\\('FE24ADC5E11E2B25288D1704ABE67A79E342ECC26064CE69C5B3177795A82264', 'hex'\\)" + replace: "NULL" + + - name: Debug webapp admin password usage + ansible.builtin.debug: + msg: | + Webapp admin password usage (vdi_tool): + - password_provided_by_secret_manager: {{ webapp_admin_password is defined }} + - password_length: {{ webapp_admin_password | length if webapp_admin_password is defined else 'N/A' }} + - hash_computed: {{ webapp_admin_hash is defined }} + when: debug + + - name: Stop and remove existing containers + ansible.builtin.command: docker stop {{ item }} + loop: + - guac_app + - guacd + - guac_db + ignore_errors: true + + - name: Remove existing containers + ansible.builtin.command: docker rm {{ item }} + loop: + - guac_app + - guacd + - guac_db + ignore_errors: true + + - name: Backup existing database data + ansible.builtin.shell: > + tar -czf /opt/guacamole-db/backup-$(date +%Y%m%d-%H%M%S).tar.gz + -C /opt/guacamole-db data/ + ignore_errors: true + + - name: Remove existing database data + ansible.builtin.file: + path: /opt/guacamole-db/data + state: absent + ignore_errors: true + + - name: Recreate database data directory + ansible.builtin.file: + path: /opt/guacamole-db/data + state: directory + owner: root + group: root + mode: '0700' + + - name: Start Guacamole PostgreSQL container (re-initialized) + ansible.builtin.command: > + docker run -d --name guac_db --network guac_net + -e POSTGRES_USER=guacamole_db + -e POSTGRES_PASSWORD={{ database_password }} + -e POSTGRES_DB=guacamole_db + -v /opt/guacamole-db/data:/var/lib/postgresql/data + -v /opt/guacamole-db/initdb:/docker-entrypoint-initdb.d + --restart always + postgres:latest + register: postgres_restart + failed_when: + - postgres_restart.rc != 0 + - "'already in use' not in postgres_restart.stderr" + + - name: Wait for PostgreSQL to be ready after re-initialization + ansible.builtin.command: docker exec guac_db pg_isready -U guacamole_db + register: postgres_ready_after_restart + until: postgres_ready_after_restart.rc == 0 + retries: 30 + delay: 2 + + - name: Check if Guacamole tables exist + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_name LIKE 'guacamole_%';" + register: table_check + changed_when: false + + - name: Debug table check + ansible.builtin.debug: + msg: | + Table check results: + - tables_found: {{ table_check.stdout_lines | length }} + - table_names: {{ table_check.stdout_lines }} + when: debug + + - name: Check if connections were created + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c + "SELECT connection_name FROM guacamole_connection;" + register: connection_check + changed_when: false + + - name: Debug connection check + ansible.builtin.debug: + msg: | + Connection check results: + - connections_found: {{ connection_check.stdout_lines | length }} + - connection_names: {{ connection_check.stdout_lines }} + when: debug + + - name: Wait for PostgreSQL via Docker exec + ansible.builtin.command: docker exec guac_db pg_isready -U guacamole_db + register: postgres_ready + until: postgres_ready.rc == 0 + retries: 30 delay: 2 + + - name: Check if connections exist (non-reinitialisation case) + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c + "SELECT connection_name FROM guacamole_connection;" + register: existing_connection_check + changed_when: false + + - name: Debug existing connection check + ansible.builtin.debug: + msg: | + Existing VNC Connections: + {{ existing_connection_check.stdout }} + when: debug + + - name: Start guacd container (force rerun or if not running) + ansible.builtin.command: > + docker run -d --name guacd --network guac_net + -p 4822:4822 + --restart always + guacamole/guacd:latest + register: guacd_start + failed_when: + - guacd_start.rc != 0 + - "'already in use' not in guacd_start.stderr" + + - name: Start Guacamole webapp container (force rerun or if not running) + ansible.builtin.command: > + docker run -d --name guac_app --network guac_net + -p {{ vdi_webapp_port }}:8080 + -e POSTGRES_HOSTNAME=guac_db + -e POSTGRES_PORT=5432 + -e POSTGRES_DATABASE=guacamole_db + -e POSTGRES_USER=guacamole_db + -e POSTGRES_PASSWORD={{ database_password }} + -e POSTGRES_AUTO_CREATE_ACCOUNTS=true + -e GUACD_HOSTNAME=guacd + -e GUACD_PORT=4822 + --restart always + guacamole/guacamole:latest + register: guac_app_start + failed_when: + - guac_app_start.rc != 0 + - "'already in use' not in guac_app_start.stderr" + + - name: Debug container status before waiting for Guacamole + ansible.builtin.command: docker ps + register: container_status_before_wait + changed_when: false + + - name: Show container status + ansible.builtin.debug: + msg: | + {{ container_status_before_wait.stdout }} + when: debug + + - name: Wait for Guacamole HTTP endpoint + ansible.builtin.wait_for: + host: 127.0.0.1 + port: "{{ vdi_webapp_port }}" + delay: 5 timeout: 60 - ignore_errors: true - when: "'guac_db' not in running_containers.stdout" + + # ============================================================================= + # UPDATE PASSWORDS AFTER DATABASE REINITIALIZATION + # ============================================================================= + + - name: Update database user password (after reinitialization) + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c + "ALTER USER guacamole_db WITH PASSWORD '{{ database_password }}';" + + - name: Update webapp admin password in database (after reinitialization) + ansible.builtin.command: > + docker exec guac_db psql -U guacamole_db -d guacamole_db -c + "UPDATE guacamole_user SET password_salt = NULL, password_hash = decode('{{ webapp_admin_password | hash('sha256') | upper }}', 'hex'), password_date = now() FROM guacamole_entity WHERE guacamole_user.entity_id = guacamole_entity.entity_id AND guacamole_entity.name = 'guacadmin' AND guacamole_entity.type = 'USER';" + + - name: Restart Guacamole webapp container (after password update) + ansible.builtin.command: docker restart guac_app + + - name: Wait for Guacamole webapp to be ready after password update restart + ansible.builtin.wait_for: + host: 127.0.0.1 + port: "{{ vdi_webapp_port }}" + delay: 5 + timeout: 60 + + - name: Debug password update completion after reinitialization + ansible.builtin.debug: + msg: | + Password update completed after database reinitialization: + - database_password_updated: True + - webapp_admin_password_updated: True + - container_restarted: True + when: debug + + - name: Obtain Guacamole API token to verify login + ansible.builtin.uri: + url: "http://localhost:{{ vdi_webapp_port }}/guacamole/api/tokens" + method: POST + body_format: form-urlencoded + body: + username: "guacadmin" + password: "{{ webapp_admin_password }}" + return_content: yes + register: guac_api_token + until: guac_api_token.status == 200 and guac_api_token.json.authToken is defined + retries: 10 + delay: 3 + + - name: Debug Guacamole API token verification + ansible.builtin.debug: + msg: | + Guacamole API token verification: + - status: {{ guac_api_token.status }} + - success: {{ guac_api_token.status == 200 }} + - token_obtained: {{ guac_api_token.json.authToken is defined }} + when: debug + + when: db_needs_init + +# ============================================================================= +# CONTAINER MANAGEMENT (IDEMPOTENT) +# ============================================================================= - name: Wait for PostgreSQL via Docker exec ansible.builtin.command: docker exec guac_db pg_isready -U guacamole_db @@ -377,7 +817,7 @@ until: postgres_ready.rc == 0 retries: 30 delay: 2 - when: "'guac_db' not in running_containers.stdout" + when: "'guac_db' in running_containers.stdout" - name: Check if connections exist (non-reinitialisation case) ansible.builtin.command: > @@ -392,7 +832,7 @@ msg: | Existing VNC Connections: {{ existing_connection_check.stdout }} - when: debug | default(false) and not db_needs_init + when: debug and not db_needs_init - name: Start guacd container (only if not running) ansible.builtin.command: > @@ -404,7 +844,7 @@ failed_when: - guacd_start.rc != 0 - "'already in use' not in guacd_start.stderr" - when: "'guacd' not in running_containers.stdout" + when: db_needs_init or (not db_needs_init and "'guacd' not in running_containers.stdout") - name: Start Guacamole webapp container (only if not running) ansible.builtin.command: > @@ -424,7 +864,7 @@ failed_when: - guac_app_start.rc != 0 - "'already in use' not in guac_app_start.stderr" - when: "'guac_app' not in running_containers.stdout" + when: db_needs_init or (not db_needs_init and "'guac_app' not in running_containers.stdout") # ============================================================================= # VERIFICATION AND CLEANUP @@ -438,7 +878,7 @@ - name: Show container status ansible.builtin.debug: msg: "{{ container_status.stdout }}" - when: debug | default(false) + when: debug - name: Wait for Guacamole HTTP endpoint ansible.builtin.wait_for: @@ -469,16 +909,16 @@ msg: | Connection verification results: {{ connection_verification.stdout }} - when: debug | default(false) + when: debug - name: Verify connection permissions were created ansible.builtin.command: > docker exec guac_db psql -U guacamole_db -d guacamole_db -c - "SELECT e.name as username, c.connection_name, cp.permission - FROM guacamole_entity e - JOIN guacamole_connection_permission cp ON e.entity_id = cp.entity_id - JOIN guacamole_connection c ON cp.connection_id = c.connection_id - WHERE e.type = 'USER' AND e.name != 'guacadmin' + "SELECT e.name as username, c.connection_name, cp.permission + FROM guacamole_entity e + JOIN guacamole_connection_permission cp ON e.entity_id = cp.entity_id + JOIN guacamole_connection c ON cp.connection_id = c.connection_id + WHERE e.type = 'USER' AND e.name != 'guacadmin' ORDER BY e.name, c.connection_name, cp.permission;" register: permission_verification changed_when: false @@ -488,45 +928,28 @@ msg: | Permission verification results: {{ permission_verification.stdout }} - when: debug | default(false) + when: debug - name: Debug fresh deployment reset logic ansible.builtin.debug: msg: | Fresh deployment detected - password reset logic skipped: - - is_fresh_deployment: {{ is_fresh_deployment | default(false) }} + - is_fresh_deployment: {{ is_fresh_deployment }} - users_needing_reset: [] (empty list) - reset_password flags will be ignored - - reset_webapp_admin_password: {{ reset_webapp_admin_password | default(false) }} (will be ignored on fresh deployment) - when: debug | default(false) and is_fresh_deployment | default(false) + - reset_webapp_admin_password: {{ reset_webapp_admin_password }} (will be ignored on fresh deployment) + when: debug and is_fresh_deployment -- name: Update webapp admin password in database (if reset requested) - ansible.builtin.command: > - docker exec guac_db psql -U guacamole_db -d guacamole_db -c - "UPDATE guacamole_user SET password_salt = NULL, password_hash = decode('{{ webapp_admin_hash }}', 'hex'), password_date = now() FROM guacamole_entity WHERE guacamole_user.entity_id = guacamole_entity.entity_id AND guacamole_entity.name = 'guacadmin' AND guacamole_entity.type = 'USER';" - when: reset_webapp_admin_password | default(false) and not (is_fresh_deployment | default(false)) - -- name: Debug webapp admin password update +- name: Debug API verification logic ansible.builtin.debug: msg: | - Webapp admin password update: - - reset_requested: {{ reset_webapp_admin_password | default(false) }} - - is_fresh_deployment: {{ is_fresh_deployment | default(false) }} - - password_updated: {{ reset_webapp_admin_password | default(false) and not (is_fresh_deployment | default(false)) }} - - password_length: {{ webapp_admin_password | length if webapp_admin_password is defined else 'N/A' }} - when: debug | default(false) and reset_webapp_admin_password | default(false) - -- name: Restart Guacamole webapp container (if webapp admin password reset) - ansible.builtin.command: docker restart guac_app - when: reset_webapp_admin_password | default(false) and not (is_fresh_deployment | default(false)) - -- name: Wait for Guacamole webapp to be ready after restart - ansible.builtin.wait_for: - host: 127.0.0.1 - port: "{{ vdi_webapp_port }}" - delay: 5 - timeout: 60 - when: reset_webapp_admin_password | default(false) and not (is_fresh_deployment | default(false)) + API verification logic: + - is_fresh_deployment: {{ is_fresh_deployment }} + - infrastructure_changed: {{ infrastructure_changed }} + - webapp_port_changed: {{ webapp_port_changed }} + - will_verify_api: {{ is_fresh_deployment or infrastructure_changed or webapp_port_changed }} + - reason: {{ 'Fresh deployment' if is_fresh_deployment else ('Infrastructure change' if infrastructure_changed else ('Webapp port change' if webapp_port_changed else 'User-only changes - API verification skipped')) }} + when: debug - name: Obtain Guacamole API token to verify login ansible.builtin.uri: @@ -538,14 +961,43 @@ password: "{{ webapp_admin_password }}" return_content: yes register: guac_api - until: - - guac_api.status == 200 - - guac_api.json.authToken is defined + until: guac_api.status == 200 and guac_api.json.authToken is defined retries: 10 delay: 3 + when: + - webapp_admin_password is defined + - (is_fresh_deployment or infrastructure_changed or webapp_port_changed) - name: Remove bootstrap SQL files unless debug is enabled ansible.builtin.file: path: /opt/guacamole-db/initdb state: absent - when: not debug | default(false) + when: not debug + +- name: VDI Tool deployment summary + ansible.builtin.debug: + msg: | + ================================================================================ + VDI TOOL DEPLOYMENT SUMMARY + ================================================================================ + + DEPLOYMENT MODE: + - is_fresh_deployment: {{ is_fresh_deployment }} + - db_needs_init: {{ db_needs_init }} + - users_changed: {{ users_changed }} + - force_rerun: {{ force_rerun }} + - reset_webapp_admin_password: {{ reset_webapp_admin_password }} + + TARGETED UPDATES APPLIED: + - New users added: {{ new_users | map(attribute='username') | list if new_users is defined and new_users | length > 0 else [] }} + - Users removed: {{ removed_users if removed_users is defined and removed_users | length > 0 else [] }} + - Password resets: {{ users_needing_reset | map(attribute='username') | list if users_needing_reset is defined and users_needing_reset | length > 0 else [] }} + - Webapp admin password reset: {{ reset_webapp_admin_password and not is_fresh_deployment }} + + CONTAINER STATUS: + - Database container: {{ 'Running' if 'guac_db' in running_containers.stdout else 'Not running' }} + - Guacd container: {{ 'Running' if 'guacd' in running_containers.stdout else 'Not running' }} + - Webapp container: {{ 'Running' if 'guac_app' in running_containers.stdout else 'Not running' }} + + ================================================================================ + when: debug diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml index d89a4fe845..a1b9349c6e 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/tasks/main.yaml @@ -22,29 +22,46 @@ ansible.builtin.import_role: name: lock_manager tasks_from: check_lock - register: lock_check_result -- name: Set role_should_run from lock check result +- name: Set role variables from lock manager facts ansible.builtin.set_fact: - role_should_run: "{{ lock_check_result.ansible_facts.role_should_run | default(true) }}" + role_should_run: "{{ role_should_run | default(false) }}" + is_fresh_deployment: "{{ is_fresh_deployment | default(false) }}" + deployment_hash: "{{ current_deployment_hash | default('none') }}" + user_hash: "{{ current_user_secrets_hash | default('none') }}" + lock_file_stat: "{{ lock_file_stat | default({}) }}" + vdi_setup_status: "{{ vdi_setup_status | default({}) }}" + users_with_port_changes: "{{ users_with_port_changes | default([]) }}" + webapp_port_changed: "{{ webapp_port_changed | default(false) }}" + vdi_resolution_changed: "{{ vdi_resolution_changed | default(false) }}" + vdi_resolution_locked_changed: "{{ vdi_resolution_locked_changed | default(false) }}" + infrastructure_changed: "{{ infrastructure_changed | default(false) }}" + new_users: "{{ new_users | default([]) }}" + removed_users: "{{ removed_users | default([]) }}" + users_needing_reset: "{{ users_needing_reset | default([]) }}" + secret_project: "{{ project_id }}" + reset_webapp_admin_password: "{{ reset_webapp_admin_password | default(false) }}" + force_rerun: "{{ force_rerun | default(false) }}" + webapp_admin_hash: "{{ webapp_admin_hash | default('') }}" -- name: Check if user secrets changed - ansible.builtin.set_fact: - user_secrets_changed: "{{ lock_check_result.ansible_facts.user_secrets_changed | default(false) }}" - -- name: Override role_should_run if user secrets changed - ansible.builtin.set_fact: - role_should_run: true - when: user_secrets_changed - -- name: Debug vdi_tool role execution reason +- name: Debug role variables ansible.builtin.debug: msg: | - VDI tool role execution: + VDI Tool role variables: - role_should_run: {{ role_should_run }} - - user_secrets_changed: {{ user_secrets_changed }} - - execution_reason: {{ 'User configuration changed' if user_secrets_changed else 'Normal role execution' }} - when: role_should_run + - is_fresh_deployment: {{ is_fresh_deployment }} + - deployment_hash: {{ deployment_hash }} + - user_hash: {{ user_hash }} + - vdi_resolution_changed: {{ vdi_resolution_changed }} + - vdi_resolution_locked_changed: {{ vdi_resolution_locked_changed }} + - infrastructure_changed: {{ infrastructure_changed }} + - execution_reason: {{ 'Fresh deployment' if is_fresh_deployment else ('Hash mismatch' if not role_should_run else 'Normal role execution') }} + when: debug + +- name: Skip VDI Tool role if not needed + ansible.builtin.debug: + msg: "Skipping VDI Tool role - already completed or not needed" + when: not role_should_run # Skip all tasks if role should not run - name: Skip vdi_tool tasks if role should not run @@ -54,21 +71,21 @@ - name: "Install Guacamole" include_tasks: guacamole.yaml - when: - - vdi_tool | lower == 'guacamole' - - role_should_run + when: + - vdi_tool | lower == 'guacamole' + - role_should_run - name: "Install NoMachine" include_tasks: nomachine.yaml - when: - - vdi_tool | lower == 'nomachine' - - role_should_run + when: + - vdi_tool | lower == 'nomachine' + - role_should_run - name: "Install Workspot" include_tasks: workspot.yaml - when: - - vdi_tool | lower == 'workspot' - - role_should_run + when: + - vdi_tool | lower == 'workspot' + - role_should_run # Mark role as completed - name: Set current role for lock manager completion diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_bootstrap.sql.j2 b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_bootstrap.sql.j2 index 7685d83532..3be0c9ca60 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_bootstrap.sql.j2 +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_bootstrap.sql.j2 @@ -11,7 +11,8 @@ FROM guacamole_connection, (VALUES ('hostname', '{{ ansible_default_ipv4.address }}'), ('port', '{{ port }}'), ('username', '{{ username }}'), - ('password', '{{ vnc_password }}') + ('password', '{{ vnc_password }}'), + ('disable-display-resize', '{{ "true" if vdi_resolution_locked else "false" }}') ) AS params(param_name, param_value) WHERE connection_name = '{{ username }}'; diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_port_update.sql.j2 b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_port_update.sql.j2 new file mode 100644 index 0000000000..6dbb21fe81 --- /dev/null +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_port_update.sql.j2 @@ -0,0 +1,5 @@ +-- Update VNC Connection Port for {{ username }} +UPDATE guacamole_connection_parameter +SET parameter_value = '{{ port }}' +WHERE connection_id = (SELECT connection_id FROM guacamole_connection WHERE connection_name = '{{ username }}') +AND parameter_name = 'port'; diff --git a/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml b/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml index feab71396d..3cea28cd3d 100644 --- a/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml +++ b/community/modules/scripts/vdi-setup/roles/vnc/tasks/main.yaml @@ -22,47 +22,225 @@ ansible.builtin.import_role: name: lock_manager tasks_from: check_lock - register: lock_check_result -- name: Set role_should_run from lock check result +- name: Set role variables from lock manager facts ansible.builtin.set_fact: - role_should_run: "{{ lock_check_result.ansible_facts.role_should_run | default(true) }}" + role_should_run: "{{ role_should_run | default(false) }}" + is_fresh_deployment: "{{ is_fresh_deployment | default(false) }}" + deployment_hash: "{{ current_deployment_hash | default('none') }}" + user_hash: "{{ current_user_secrets_hash | default('none') }}" + lock_file_stat: "{{ lock_file_stat | default({}) }}" + vdi_setup_status: "{{ vdi_setup_status | default({}) }}" + users_with_port_changes: "{{ users_with_port_changes | default([]) }}" + removed_users: "{{ removed_users | default([]) }}" + vdi_resolution_changed: "{{ vdi_resolution_changed | default(false) }}" + vdi_resolution_locked_changed: "{{ vdi_resolution_locked_changed | default(false) }}" -- name: Check if user secrets changed - ansible.builtin.set_fact: - user_secrets_changed: "{{ lock_check_result.ansible_facts.user_secrets_changed | default(false) }}" +- name: Debug role variables + ansible.builtin.debug: + msg: | + VNC role variables: + - role_should_run: {{ role_should_run }} + - is_fresh_deployment: {{ is_fresh_deployment }} + - deployment_hash: {{ deployment_hash }} + - user_hash: {{ user_hash }} + - vdi_resolution_changed: {{ vdi_resolution_changed }} + - vdi_resolution_locked_changed: {{ vdi_resolution_locked_changed }} + - execution_reason: {{ 'Fresh deployment' if is_fresh_deployment else ('Hash mismatch' if not role_should_run else 'Normal role execution') }} + when: debug -- name: Override role_should_run if user secrets changed - ansible.builtin.set_fact: - role_should_run: true - when: user_secrets_changed +- name: Debug resolution change detection + ansible.builtin.debug: + msg: | + Resolution change detection: + - vdi_resolution_changed: {{ vdi_resolution_changed }} + - current_resolution: {{ vdi_resolution }} + - previous_resolution: {{ vdi_setup_status.vdi_resolution | default('unknown') }} + - will_restart_services: {{ vdi_resolution_changed }} + when: debug and vdi_resolution_changed -- name: Debug vnc role execution reason +- name: Debug resolution locked change detection ansible.builtin.debug: msg: | - VNC role execution: - - role_should_run: {{ role_should_run }} - - user_secrets_changed: {{ user_secrets_changed }} - - execution_reason: {{ 'User configuration changed' if user_secrets_changed else 'Normal role execution' }} - when: role_should_run + Resolution locked change detection: + - vdi_resolution_locked_changed: {{ vdi_resolution_locked_changed }} + - current_resolution_locked: {{ vdi_resolution_locked }} + - previous_resolution_locked: {{ vdi_setup_status.vdi_resolution_locked | default('unknown') }} + - will_update_connections: {{ vdi_resolution_locked_changed }} + when: debug and vdi_resolution_locked_changed -# Skip all tasks if role should not run -- name: Skip vnc tasks if role should not run +- name: Skip VNC role if not needed ansible.builtin.debug: - msg: "Skipping vnc role - already completed or not needed" + msg: "Skipping VNC role - already completed or not needed" when: not role_should_run -- name: "Install TigerVNC" - include_tasks: tigervnc.yaml - when: - - vnc_flavor | lower == 'tigervnc' - - role_should_run - -- name: "Install TightVNC" - include_tasks: tightvnc.yaml - when: - - vnc_flavor | lower == 'tightvnc' - - role_should_run +# Run VNC tasks only if needed +- name: Run VNC tasks + block: + + - name: "Install TigerVNC" + include_tasks: tigervnc.yaml + when: vnc_flavor | lower == 'tigervnc' + + - name: "Install TightVNC" + include_tasks: tightvnc.yaml + when: vnc_flavor | lower == 'tightvnc' + + # ============================================================================= + # USER REMOVAL HANDLING + # ============================================================================= + + - name: Handle removed users (stop VNC services and lock accounts) + block: + + - name: Stop VNC services for removed users + ansible.builtin.systemd: + name: "vncserver@{{ item }}" + state: stopped + enabled: no + loop: "{{ removed_users }}" + when: removed_users is defined and removed_users | length > 0 + ignore_errors: true + + - name: Kill X server processes for removed users + ansible.builtin.shell: | + pkill -f "Xvnc.*:{{ item }}" || true + sleep 2 + loop: "{{ removed_users }}" + when: removed_users is defined and removed_users | length > 0 + ignore_errors: true + + - name: Lock local user accounts for removed users + ansible.builtin.user: + name: "{{ item }}" + password: "!" + loop: "{{ removed_users }}" + when: removed_users is defined and removed_users | length > 0 + + - name: Debug user removal actions + ansible.builtin.debug: + msg: | + User removal actions completed: + {% if removed_users is defined and removed_users | length > 0 %} + - Users locked: {{ removed_users }} + - VNC services stopped and disabled + - Local accounts locked with '!' password + {% else %} + - No users to remove + {% endif %} + when: debug + + when: removed_users is defined and removed_users | length > 0 + + # ============================================================================= + # PORT CHANGE HANDLING + # ============================================================================= + + - name: Handle port changes for existing users + block: + - name: Stop VNC services for users with port changes + ansible.builtin.systemd: + name: "vncserver@:{{ item.port - 5900 }}" + state: stopped + loop: "{{ users_with_port_changes }}" + when: users_with_port_changes is defined and users_with_port_changes | length > 0 + ignore_errors: true + + - name: Kill X server processes for users with port changes (old display) + ansible.builtin.shell: | + pkill -f "Xvnc.*:{{ (vdi_setup_status.user_ports[item.username] - 5900) }}" || true + sleep 2 + loop: "{{ users_with_port_changes }}" + when: + - users_with_port_changes is defined and users_with_port_changes | length > 0 + - vdi_setup_status.user_ports[item.username] is defined + ignore_errors: true + + - name: Update VNC users mapping for port changes + ansible.builtin.template: + src: vncserver.users.j2 + dest: /etc/tigervnc/vncserver.users + mode: '0644' + vars: + guac_map: "{{ vdi_users }}" + when: users_with_port_changes is defined and users_with_port_changes | length > 0 + + - name: Start VNC services for users with port changes + ansible.builtin.systemd: + name: "vncserver@:{{ item.port - 5900 }}" + state: started + enabled: yes + loop: "{{ users_with_port_changes }}" + when: users_with_port_changes is defined and users_with_port_changes | length > 0 + + - name: Debug port change actions + ansible.builtin.debug: + msg: | + Port change actions completed: + {% if users_with_port_changes is defined and users_with_port_changes | length > 0 %} + - Users with port changes: {{ users_with_port_changes | map(attribute='username') | list }} + - VNC services stopped, reconfigured, and restarted + {% else %} + - No port changes detected + {% endif %} + when: debug + + when: users_with_port_changes is defined and users_with_port_changes | length > 0 + + # ============================================================================= + # RESOLUTION CHANGE HANDLING + # ============================================================================= + + - name: Handle resolution changes for all users + block: + - name: Stop all VNC services for resolution change + ansible.builtin.systemd: + name: "vncserver@:{{ item.port - 5900 }}" + state: stopped + loop: "{{ vdi_users }}" + when: vdi_resolution_changed + ignore_errors: true + + - name: Kill all X server processes for resolution change + ansible.builtin.shell: | + pkill -f "Xvnc.*:{{ item.port - 5900 }}" || true + sleep 2 + loop: "{{ vdi_users }}" + when: vdi_resolution_changed + ignore_errors: true + + - name: Update VNC users mapping for resolution change + ansible.builtin.template: + src: vncserver.users.j2 + dest: /etc/tigervnc/vncserver.users + mode: '0644' + vars: + guac_map: "{{ vdi_users }}" + when: vdi_resolution_changed + + - name: Start all VNC services for resolution change + ansible.builtin.systemd: + name: "vncserver@:{{ item.port - 5900 }}" + state: started + enabled: yes + loop: "{{ vdi_users }}" + when: vdi_resolution_changed + + - name: Debug resolution change actions + ansible.builtin.debug: + msg: | + Resolution change actions completed: + {% if vdi_resolution_changed %} + - Resolution changed from {{ vdi_setup_status.vdi_resolution | default('unknown') }} to {{ vdi_resolution }} + - All VNC services stopped, reconfigured, and restarted + {% else %} + - No resolution changes detected + {% endif %} + when: debug + + when: vdi_resolution_changed + + when: role_should_run # Mark role as completed - name: Set current role for lock manager completion diff --git a/community/modules/scripts/vdi-setup/roles/vnc/tasks/tigervnc.yaml b/community/modules/scripts/vdi-setup/roles/vnc/tasks/tigervnc.yaml index 0401806493..2db0fe4f0a 100644 --- a/community/modules/scripts/vdi-setup/roles/vnc/tasks/tigervnc.yaml +++ b/community/modules/scripts/vdi-setup/roles/vnc/tasks/tigervnc.yaml @@ -67,3 +67,18 @@ state: started when: ansible_os_family == "RedHat" loop: "{{ vdi_users_updated }}" + +# Restart all VNC services to ensure they pick up any password changes +- name: Restart all VNC services (Rocky) + systemd: + name: "vncserver@:{{ item.display_number }}" + state: restarted + when: ansible_os_family == "RedHat" + loop: "{{ vdi_users_updated }}" + +- name: Restart all VNC services (Ubuntu/Debian) + systemd: + name: "vncserver@{{ item.username }}" + state: restarted + when: ansible_distribution in ["Ubuntu", "Debian"] + loop: "{{ vdi_users_updated }}" diff --git a/community/modules/scripts/vdi-setup/roles/vnc/tasks/vnc_user_config.yaml b/community/modules/scripts/vdi-setup/roles/vnc/tasks/vnc_user_config.yaml index 206551b270..85107079ab 100644 --- a/community/modules/scripts/vdi-setup/roles/vnc/tasks/vnc_user_config.yaml +++ b/community/modules/scripts/vdi-setup/roles/vnc/tasks/vnc_user_config.yaml @@ -37,9 +37,8 @@ - name: Write VNC password file for {{ vdi_user.username }} ansible.builtin.shell: | echo "{{ vdi_user.vncserver_password }}" | vncpasswd -f > "/home/{{ vdi_user.username }}/.vnc/passwd" - args: - creates: "/home/{{ vdi_user.username }}/.vnc/passwd" become_user: "{{ vdi_user.username }}" + register: vnc_password_result - name: Fix passwd file permissions ansible.builtin.file: diff --git a/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl b/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl index 5b90d80ed1..b123719c3f 100644 --- a/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl +++ b/community/modules/scripts/vdi-setup/templates/vars.yaml.tftpl @@ -8,6 +8,7 @@ vdi_tool: ${vdi_tool} vdi_user_group: ${vdi_user_group} vdi_webapp_port: ${vdi_webapp_port} vdi_resolution: ${vdi_resolution} +vdi_resolution_locked: ${vdi_resolution_locked} vdi_bucket_name: ${vdi_bucket_name} zone: ${zone} debug: ${debug} diff --git a/community/modules/scripts/vdi-setup/validation.tf b/community/modules/scripts/vdi-setup/validation.tf index cd352b54cc..c5d2b26221 100644 --- a/community/modules/scripts/vdi-setup/validation.tf +++ b/community/modules/scripts/vdi-setup/validation.tf @@ -34,6 +34,11 @@ resource "terraform_data" "input_validation" { error_message = "vdi_resolution must be in the form WIDTHxHEIGHT (e.g. 1920x1080)." } + precondition { + condition = var.vdi_resolution_locked == true || var.vdi_resolution_locked == false + error_message = "vdi_resolution_locked must be a boolean value (true/false)." + } + precondition { condition = var.user_provision == "local_users" || length(var.vdi_users) == 0 error_message = "vdi_users may only be set when user_provision = local_users." @@ -58,5 +63,10 @@ resource "terraform_data" "input_validation" { ]) error_message = "reset_password must be a boolean value (true/false) or null." } + + precondition { + condition = length(distinct([for user in var.vdi_users : user.port])) == length(var.vdi_users) + error_message = "All VDI users must have unique ports to prevent conflicts." + } } } diff --git a/community/modules/scripts/vdi-setup/variables.tf b/community/modules/scripts/vdi-setup/variables.tf index ca959ba532..96549d3159 100644 --- a/community/modules/scripts/vdi-setup/variables.tf +++ b/community/modules/scripts/vdi-setup/variables.tf @@ -67,9 +67,15 @@ variable "vdi_resolution" { default = "1920x1080" } +variable "vdi_resolution_locked" { + type = bool + description = "Disable resize of remote display in Guacamole connections. When true, VDI displays at native resolution without browser scaling." + default = true +} + variable "vdi_webapp_port" { type = string - description = "Port to serve the Webapp interface from if applicable" + description = "Port to serve the Webapp interface from if applicable (note: containers will be recreated if changed)" default = "8080" } diff --git a/community/modules/scripts/vdi-setup/versions.tf b/community/modules/scripts/vdi-setup/versions.tf index 6024b738a7..64da35c590 100644 --- a/community/modules/scripts/vdi-setup/versions.tf +++ b/community/modules/scripts/vdi-setup/versions.tf @@ -25,5 +25,9 @@ terraform { source = "hashicorp/google" version = ">= 3.83" } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } } } From 5351e345ade91fb2ddf14bea353ebde7adaf9135 Mon Sep 17 00:00:00 2001 From: Scott Gordon Date: Thu, 28 Aug 2025 12:10:16 +0100 Subject: [PATCH 19/21] pre-commit changes --- .../vdi_monitor/templates/vdi-monitor.service.j2 | 16 ++++++++++++++++ .../guacamole/connection_bootstrap.sql.j2 | 16 ++++++++++++++++ .../guacamole/connection_port_update.sql.j2 | 16 ++++++++++++++++ .../templates/guacamole/user_bootstrap.sql.j2 | 16 ++++++++++++++++ .../vdi-setup/roles/vnc/templates/config.j2 | 16 ++++++++++++++++ .../roles/vnc/templates/vncserver.users.j2 | 16 ++++++++++++++++ .../vnc/templates/vncserver@.service.rocky.j2 | 16 ++++++++++++++++ .../vnc/templates/vncserver@.service.ubuntu.j2 | 16 ++++++++++++++++ 8 files changed, 128 insertions(+) diff --git a/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor.service.j2 b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor.service.j2 index 58d9449b6b..6f4016df3b 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor.service.j2 +++ b/community/modules/scripts/vdi-setup/roles/vdi_monitor/templates/vdi-monitor.service.j2 @@ -1,3 +1,19 @@ +{# +Copyright 2025 "Google LLC" + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} + [Unit] Description=VDI Configuration Monitor After=network.target diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_bootstrap.sql.j2 b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_bootstrap.sql.j2 index 3be0c9ca60..0835ce73f6 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_bootstrap.sql.j2 +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_bootstrap.sql.j2 @@ -1,3 +1,19 @@ +{# +Copyright 2025 "Google LLC" + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} + -- VNC Connection for {{ username }} INSERT INTO guacamole_connection (connection_name, protocol) VALUES ('{{ username }}', 'vnc'); diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_port_update.sql.j2 b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_port_update.sql.j2 index 6dbb21fe81..82dc83d7e8 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_port_update.sql.j2 +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/connection_port_update.sql.j2 @@ -1,3 +1,19 @@ +{# +Copyright 2025 "Google LLC" + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} + -- Update VNC Connection Port for {{ username }} UPDATE guacamole_connection_parameter SET parameter_value = '{{ port }}' diff --git a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/user_bootstrap.sql.j2 b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/user_bootstrap.sql.j2 index 784d949d4e..aeda82561f 100644 --- a/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/user_bootstrap.sql.j2 +++ b/community/modules/scripts/vdi-setup/roles/vdi_tool/templates/guacamole/user_bootstrap.sql.j2 @@ -1,3 +1,19 @@ +{# +Copyright 2025 "Google LLC" + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} + -- Create Guacamole user for {{ username }} INSERT INTO guacamole_entity (name, type) VALUES ('{{ username }}', 'USER'); diff --git a/community/modules/scripts/vdi-setup/roles/vnc/templates/config.j2 b/community/modules/scripts/vdi-setup/roles/vnc/templates/config.j2 index 605ec72eca..9c379307fb 100644 --- a/community/modules/scripts/vdi-setup/roles/vnc/templates/config.j2 +++ b/community/modules/scripts/vdi-setup/roles/vnc/templates/config.j2 @@ -1,2 +1,18 @@ +{# +Copyright 2025 "Google LLC" + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} + session=xfce geometry={{ vdi_resolution }} diff --git a/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver.users.j2 b/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver.users.j2 index cbdab2dbc0..fa42c3d857 100644 --- a/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver.users.j2 +++ b/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver.users.j2 @@ -1,3 +1,19 @@ +{# +Copyright 2025 "Google LLC" + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} + {% for user in guac_map %} :{{ user.port - 5900 }}={{ user.username }} {% endfor %} diff --git a/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.rocky.j2 b/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.rocky.j2 index a7cc3cc10a..be88533884 100644 --- a/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.rocky.j2 +++ b/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.rocky.j2 @@ -1,3 +1,19 @@ +{# +Copyright 2025 "Google LLC" + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} + [Unit] Description=TigerVNC server for user %i After=syslog.target network.target diff --git a/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.ubuntu.j2 b/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.ubuntu.j2 index 1df046f446..6db778debd 100644 --- a/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.ubuntu.j2 +++ b/community/modules/scripts/vdi-setup/roles/vnc/templates/vncserver@.service.ubuntu.j2 @@ -1,3 +1,19 @@ +{# +Copyright 2025 "Google LLC" + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} + [Unit] Description=TigerVNC server for user %i After=network.target From ad8a7862dbfb64f5c9473186dc9ba21116e781ab Mon Sep 17 00:00:00 2001 From: Scott Gordon Date: Thu, 28 Aug 2025 12:30:53 +0100 Subject: [PATCH 20/21] vdi module - precommit tweaks --- .../lock_manager/templates/lock_file.yaml.j2 | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 b/community/modules/scripts/vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 index eac4765c08..600037159f 100644 --- a/community/modules/scripts/vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 +++ b/community/modules/scripts/vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 @@ -1,3 +1,19 @@ +{# +Copyright 2025 "Google LLC" + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} + # VDI Setup Lock File # Generated on {{ ansible_date_time.iso8601 }} @@ -25,4 +41,4 @@ vdi_setup_status: vnc: false vdi_tool: false vdi_monitor: false - + \ No newline at end of file From 77e6f7863a72b1dbbe61d4f3fa37bb656260f398 Mon Sep 17 00:00:00 2001 From: Scott Gordon Date: Thu, 28 Aug 2025 14:27:16 +0100 Subject: [PATCH 21/21] precommit --- .../vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/community/modules/scripts/vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 b/community/modules/scripts/vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 index 600037159f..a87c5e9e30 100644 --- a/community/modules/scripts/vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 +++ b/community/modules/scripts/vdi-setup/roles/lock_manager/templates/lock_file.yaml.j2 @@ -40,5 +40,4 @@ vdi_setup_status: user_provision: false vnc: false vdi_tool: false - vdi_monitor: false - \ No newline at end of file + vdi_monitor: false